Projects
home:Jingwiw:MultiMedia
Mesa
_service:tar_scm:llvmpipe-add-an-implementation...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:tar_scm:llvmpipe-add-an-implementation-with-llvm-orcjit.patch of Package Mesa
From 653d2fce91f09904e08c55647ca0bb901e402617 Mon Sep 17 00:00:00 2001 From: Alex Fan <alex.fan.q@gmail.com> Date: Tue, 28 Jun 2022 15:51:24 +0000 Subject: [PATCH] llvmpipe: add an implementation with llvm orcjit no per-func perf nor disasm is implemented yet. disk cache is also left empty --- diff --git a/src/gallium/auxiliary/draw/draw_context.c b/src/gallium/auxiliary/draw/draw_context.c index 9ddf91b..e3138cb 100644 --- a/src/gallium/auxiliary/draw/draw_context.c +++ b/src/gallium/auxiliary/draw/draw_context.c @@ -77,7 +77,11 @@ draw_create_context(struct pipe_context *pipe, void *context, #ifdef DRAW_LLVM_AVAILABLE if (try_llvm && draw_get_option_use_llvm()) { +#if GALLIVM_USE_ORCJIT == 1 + draw->llvm = draw_llvm_create(draw, (LLVMOrcThreadSafeContextRef)context); +#else draw->llvm = draw_llvm_create(draw, (LLVMContextRef)context); +#endif } #endif diff --git a/src/gallium/auxiliary/draw/draw_llvm.c b/src/gallium/auxiliary/draw/draw_llvm.c index 4e8b001..58ea8fd 100644 --- a/src/gallium/auxiliary/draw/draw_llvm.c +++ b/src/gallium/auxiliary/draw/draw_llvm.c @@ -384,8 +384,13 @@ get_vertex_header_ptr_type(struct draw_llvm_variant *variant) /** * Create per-context LLVM info. */ +#if GALLIVM_USE_ORCJIT == 1 +struct draw_llvm * +draw_llvm_create(struct draw_context *draw, LLVMOrcThreadSafeContextRef context) +#else struct draw_llvm * draw_llvm_create(struct draw_context *draw, LLVMContextRef context) +#endif { struct draw_llvm *llvm; @@ -398,6 +403,16 @@ draw_llvm_create(struct draw_context *draw, LLVMContextRef context) llvm->draw = draw; +#if GALLIVM_USE_ORCJIT == 1 + llvm->_ts_context = context; + if (!llvm->_ts_context) { + llvm->_ts_context = LLVMOrcCreateNewThreadSafeContext(); + llvm->context_owned = true; + } + if (!llvm->_ts_context) + goto fail; + llvm->context = LLVMOrcThreadSafeContextGetContext(llvm->_ts_context); +#else llvm->context = context; if (!llvm->context) { llvm->context = LLVMContextCreate(); @@ -410,6 +425,7 @@ draw_llvm_create(struct draw_context *draw, LLVMContextRef context) } if (!llvm->context) goto fail; +#endif llvm->nr_variants = 0; list_inithead(&llvm->vs_variants_list.list); @@ -437,9 +453,16 @@ fail: void draw_llvm_destroy(struct draw_llvm *llvm) { +#if GALLIVM_USE_ORCJIT == 1 + if (llvm->context_owned) + LLVMOrcDisposeThreadSafeContext(llvm->_ts_context); + llvm->_ts_context = NULL; + llvm->context = NULL; +#else if (llvm->context_owned) LLVMContextDispose(llvm->context); llvm->context = NULL; +#endif /* XXX free other draw_llvm data? */ FREE(llvm); @@ -513,7 +536,11 @@ draw_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_vs_jit_types(variant); @@ -532,8 +559,13 @@ draw_llvm_create_variant(struct draw_llvm *llvm, gallivm_compile_module(variant->gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_func = (draw_jit_vert_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else variant->jit_func = (draw_jit_vert_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -1633,6 +1665,10 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant) variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); for (i = 0; i < num_arg_types; ++i) @@ -2244,6 +2280,11 @@ draw_llvm_destroy_variant(struct draw_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_variants--; + +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -2353,6 +2394,10 @@ draw_gs_llvm_generate(struct draw_llvm *llvm, variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); @@ -2518,7 +2563,11 @@ draw_gs_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_gs_jit_types(variant); @@ -2529,8 +2578,13 @@ draw_gs_llvm_create_variant(struct draw_llvm *llvm, gallivm_compile_module(variant->gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_func = (draw_gs_jit_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else variant->jit_func = (draw_gs_jit_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -2563,6 +2617,10 @@ draw_gs_llvm_destroy_variant(struct draw_gs_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_gs_variants--; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -2936,6 +2994,10 @@ draw_tcs_llvm_generate(struct draw_llvm *llvm, variant_coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); LLVMSetFunctionCallConv(variant_coro, LLVMCCallConv); @@ -3171,8 +3233,11 @@ draw_tcs_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } - +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_tcs_jit_types(variant); @@ -3183,10 +3248,18 @@ draw_tcs_llvm_create_variant(struct draw_llvm *llvm, draw_tcs_llvm_generate(llvm, variant); +#if GALLIVM_USE_ORCJIT == 1 + lp_build_coro_add_malloc_hooks(variant->gallivm); + + gallivm_compile_module(variant->gallivm); + variant->jit_func = (draw_tcs_jit_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else gallivm_compile_module(variant->gallivm); variant->jit_func = (draw_tcs_jit_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -3219,6 +3292,10 @@ draw_tcs_llvm_destroy_variant(struct draw_tcs_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_tcs_variants--; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -3501,6 +3578,10 @@ draw_tes_llvm_generate(struct draw_llvm *llvm, variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); for (i = 0; i < ARRAY_SIZE(arg_types); ++i) @@ -3691,7 +3772,11 @@ draw_tes_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_tes_jit_types(variant); @@ -3707,8 +3792,13 @@ draw_tes_llvm_create_variant(struct draw_llvm *llvm, gallivm_compile_module(variant->gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_func = (draw_tes_jit_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else variant->jit_func = (draw_tes_jit_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -3741,6 +3831,10 @@ draw_tes_llvm_destroy_variant(struct draw_tes_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_tes_variants--; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } diff --git a/src/gallium/auxiliary/draw/draw_llvm.h b/src/gallium/auxiliary/draw/draw_llvm.h index 2f66c80..dfc818e 100644 --- a/src/gallium/auxiliary/draw/draw_llvm.h +++ b/src/gallium/auxiliary/draw/draw_llvm.h @@ -42,6 +42,11 @@ #include "pipe/p_context.h" #include "util/list.h" +#define GALLIVM_USE_ORCJIT 1 + +#if GALLIVM_USE_ORCJIT == 1 +#include <llvm-c/Orc.h> +#endif struct draw_llvm; struct llvm_vertex_shader; @@ -401,6 +406,9 @@ struct draw_llvm_variant LLVMTypeRef vertex_header_ptr_type; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_jit_vert_func jit_func; struct llvm_vertex_shader *shader; @@ -434,6 +442,9 @@ struct draw_gs_llvm_variant LLVMValueRef io_ptr; LLVMValueRef num_prims; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_gs_jit_func jit_func; struct llvm_geometry_shader *shader; @@ -460,6 +471,9 @@ struct draw_tcs_llvm_variant /* LLVMValueRef io_ptr; */ LLVMValueRef num_prims; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_tcs_jit_func jit_func; struct llvm_tess_ctrl_shader *shader; @@ -490,6 +504,9 @@ struct draw_tes_llvm_variant LLVMValueRef io_ptr; LLVMValueRef num_prims; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_tes_jit_func jit_func; struct llvm_tess_eval_shader *shader; @@ -542,6 +559,9 @@ struct draw_llvm { struct draw_context *draw; LLVMContextRef context; +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef _ts_context; +#endif bool context_owned; struct draw_vs_jit_context vs_jit_context; @@ -587,8 +607,13 @@ llvm_tess_eval_shader(struct draw_tess_eval_shader *tes) return (struct llvm_tess_eval_shader *)tes; } +#if GALLIVM_USE_ORCJIT == 1 struct draw_llvm * -draw_llvm_create(struct draw_context *draw, LLVMContextRef llvm_context); +draw_llvm_create(struct draw_context *draw, LLVMOrcThreadSafeContextRef context); +#else +struct draw_llvm * +draw_llvm_create(struct draw_context *draw, LLVMContextRef context); +#endif void draw_llvm_destroy(struct draw_llvm *llvm); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_coro.c b/src/gallium/auxiliary/gallivm/lp_bld_coro.c index 0214dcf..f5a2b31 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_coro.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_coro.c @@ -156,12 +156,14 @@ coro_free(char *ptr) void lp_build_coro_add_malloc_hooks(struct gallivm_state *gallivm) { +#if GALLIVM_USE_ORCJIT == 0 assert(gallivm->engine); +#endif assert(gallivm->coro_malloc_hook); assert(gallivm->coro_free_hook); - LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_malloc_hook, coro_malloc); - LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_free_hook, coro_free); + gallivm_add_global_mapping(gallivm, gallivm->coro_malloc_hook, coro_malloc); + gallivm_add_global_mapping(gallivm, gallivm->coro_free_hook, coro_free); } void lp_build_coro_declare_malloc_hooks(struct gallivm_state *gallivm) diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.c b/src/gallium/auxiliary/gallivm/lp_bld_init.c index cd2108f..41a6257 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_init.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_init.c @@ -515,6 +515,11 @@ gallivm_destroy(struct gallivm_state *gallivm) FREE(gallivm); } +void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr) +{ + LLVMAddGlobalMapping(gallivm->engine, sym, addr); +} /** * Validate a function. @@ -670,7 +675,7 @@ gallivm_compile_module(struct gallivm_state *gallivm) ++gallivm->compiled; lp_init_printf_hook(gallivm); - LLVMAddGlobalMapping(gallivm->engine, gallivm->debug_printf_hook, debug_printf); + gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook, debug_printf); lp_init_clock_hook(gallivm); LLVMAddGlobalMapping(gallivm->engine, gallivm->get_time_hook, os_time_get_nano); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.h b/src/gallium/auxiliary/gallivm/lp_bld_init.h index 418921c..294d608 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_init.h +++ b/src/gallium/auxiliary/gallivm/lp_bld_init.h @@ -29,11 +29,16 @@ #ifndef LP_BLD_INIT_H #define LP_BLD_INIT_H +#define GALLIVM_USE_ORCJIT 1 #include "util/compiler.h" #include "util/u_pointer.h" // for func_pointer #include "lp_bld.h" +#if GALLIVM_USE_ORCJIT == 1 +#include <llvm-c/Orc.h> +#else #include <llvm-c/ExecutionEngine.h> +#endif #ifdef __cplusplus extern "C" { @@ -44,18 +49,27 @@ struct gallivm_state { char *module_name; LLVMModuleRef module; - LLVMExecutionEngineRef engine; LLVMTargetDataRef target; +#if GALLIVM_USE_ORCJIT == 1 + /* own this->module */ + LLVMOrcThreadSafeContextRef _ts_context; + /* each module is in its own jitdylib */ + LLVMOrcJITDylibRef _per_module_jd; +#else + LLVMExecutionEngineRef engine; #if GALLIVM_USE_NEW_PASS == 0 LLVMPassManagerRef passmgr; #if GALLIVM_HAVE_CORO == 1 LLVMPassManagerRef cgpassmgr; #endif #endif - LLVMContextRef context; - LLVMBuilderRef builder; LLVMMCJITMemoryManagerRef memorymgr; struct lp_generated_code *code; +#endif + + LLVMContextRef context; + LLVMBuilderRef builder; + struct lp_cached_code *cache; unsigned compiled; LLVMValueRef coro_malloc_hook; @@ -77,10 +91,15 @@ lp_build_init_native_width(void); bool lp_build_init(void); - +#if GALLIVM_USE_ORCJIT == 1 +struct gallivm_state * +gallivm_create(const char *name, LLVMOrcThreadSafeContextRef context, + struct lp_cached_code *cache); +#else struct gallivm_state * gallivm_create(const char *name, LLVMContextRef context, struct lp_cached_code *cache); +#endif void gallivm_destroy(struct gallivm_state *gallivm); @@ -93,11 +112,25 @@ gallivm_verify_function(struct gallivm_state *gallivm, LLVMValueRef func); void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr); + +/** + * for ORCJIT, after this function gets called, all access and modification to + * module and any structure associated to it should be avoided, + * as module has been moved into ORCJIT and may be recycled + */ +void gallivm_compile_module(struct gallivm_state *gallivm); +#if GALLIVM_USE_ORCJIT == 1 +func_pointer +gallivm_jit_function(struct gallivm_state *gallivm, + const char *func_name); +#else func_pointer gallivm_jit_function(struct gallivm_state *gallivm, LLVMValueRef func); +#endif unsigned gallivm_get_perf_flags(void); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp b/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp new file mode 100644 index 0000000..8ea4df7 --- /dev/null +++ b/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp @@ -0,0 +1,905 @@ +#include "util/detect.h" +#include "util/u_cpu_detect.h" +#include "util/u_debug.h" +#include "util/os_time.h" +#include "lp_bld.h" +#include "lp_bld_debug.h" +#include "lp_bld_init.h" + +#include <llvm/Config/llvm-config.h> +#include <llvm-c/Core.h> +#include <llvm-c/Orc.h> +#include <llvm-c/LLJIT.h> +#include <llvm-c/TargetMachine.h> +#include <llvm-c/Support.h> + +#include <llvm-c/Analysis.h> +#include <llvm-c/Transforms/Scalar.h> +#if LLVM_VERSION_MAJOR >= 7 +#include <llvm-c/Transforms/Utils.h> +#endif +#include <llvm-c/BitWriter.h> +#if GALLIVM_USE_NEW_PASS == 1 +#include <llvm-c/Transforms/PassBuilder.h> +#elif GALLIVM_HAVE_CORO == 1 +#if LLVM_VERSION_MAJOR <= 8 && (defined(PIPE_ARCH_AARCH64) || defined (PIPE_ARCH_ARM) || defined(PIPE_ARCH_S390) || defined(PIPE_ARCH_MIPS64)) +#include <llvm-c/Transforms/IPO.h> +#endif +#include <llvm-c/Transforms/Coroutines.h> +#endif + +#include <llvm/ADT/StringMap.h> +#include <llvm/ExecutionEngine/Orc/LLJIT.h> +#include <llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h> +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include <llvm/Target/TargetMachine.h> +#include <llvm/Support/TargetSelect.h> +#include <llvm/Support/Casting.h> +#include <llvm/Support/Host.h> +#include <llvm/Support/CBindingWrapping.h> +#if LLVM_USE_INTEL_JITEVENTS +#include <llvm/ExecutionEngine/JITEventListener.h> +#endif + +/* conflict with ObjectLinkingLayer.h */ +#include "util/u_memory.h" + +#if (defined(_WIN32) && LLVM_VERSION_MAJOR >= 15) +/* use ObjectLinkingLayer (JITLINK backend) */ +#define USE_JITLINK +#endif +/* else use old RTDyldObjectLinkingLayer (RuntimeDyld backend) */ + +unsigned lp_native_vector_width; + +unsigned gallivm_perf = 0; + +static const struct debug_named_value lp_bld_perf_flags[] = { + { "brilinear", GALLIVM_PERF_BRILINEAR, "enable brilinear optimization" }, + { "rho_approx", GALLIVM_PERF_RHO_APPROX, "enable rho_approx optimization" }, + { "no_quad_lod", GALLIVM_PERF_NO_QUAD_LOD, "disable quad_lod optimization" }, + { "no_aos_sampling", GALLIVM_PERF_NO_AOS_SAMPLING, "disable aos sampling optimization" }, + { "nopt", GALLIVM_PERF_NO_OPT, "disable optimization passes to speed up shader compilation" }, + DEBUG_NAMED_VALUE_END +}; + +unsigned gallivm_debug = 0; + +static const struct debug_named_value lp_bld_debug_flags[] = { + { "tgsi", GALLIVM_DEBUG_TGSI, NULL }, + { "ir", GALLIVM_DEBUG_IR, NULL }, + { "asm", GALLIVM_DEBUG_ASM, NULL }, + { "perf", GALLIVM_DEBUG_PERF, NULL }, + { "gc", GALLIVM_DEBUG_GC, NULL }, + { "dumpbc", GALLIVM_DEBUG_DUMP_BC, NULL }, + DEBUG_NAMED_VALUE_END +}; + +DEBUG_GET_ONCE_FLAGS_OPTION(gallivm_debug, "GALLIVM_DEBUG", lp_bld_debug_flags, 0) + +struct lp_cached_code { + void *data; + size_t data_size; + bool dont_cache; + void *jit_obj_cache; +}; + +namespace { + +class LPJit; + +class LLVMEnsureMultithreaded { +public: + LLVMEnsureMultithreaded() + { + if (!LLVMIsMultithreaded()) { + LLVMStartMultithreaded(); + } + } +}; + +LLVMEnsureMultithreaded lLVMEnsureMultithreaded; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::ThreadSafeContext, + LLVMOrcThreadSafeContextRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::IRTransformLayer, + LLVMOrcIRTransformLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITDylib, LLVMOrcJITDylibRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITTargetMachineBuilder, + LLVMOrcJITTargetMachineBuilderRef) +LLVMTargetMachineRef wrap(const llvm::TargetMachine *P) { + return reinterpret_cast<LLVMTargetMachineRef>(const_cast<llvm::TargetMachine*>(P)); +} + +llvm::ExitOnError ExitOnErr; + +inline const char* get_module_name(LLVMModuleRef mod) { + using llvm::Module; + return llvm::unwrap(mod)->getModuleIdentifier().c_str(); +} + +once_flag init_lpjit_once_flag = ONCE_FLAG_INIT; + +/* A JIT singleton built upon LLJIT */ +class LPJit +{ +public: + static LPJit* get_instance() { + call_once(&init_lpjit_once_flag, init_lpjit); + return jit; + } + + gallivm_state *find_gallivm_state(LLVMModuleRef mod) { +#if DEBUG + using llvm::Module; + auto I = gallivm_modules.find(llvm::unwrap(mod)->getModuleIdentifier()); + if (I == gallivm_modules.end()) { + debug_printf("No gallivm state found for module: %s", get_module_name(mod)); + return NULL; + } + return I->second; +#endif + return NULL; + } + + static char *get_unique_name(const char* name) { + LPJit* jit = get_instance(); + size_t size = name == NULL? 16: strlen(name) + 16; + char *name_uniq = (char *)MALLOC(size); + if (!name_uniq) { + return NULL; + } + do { + snprintf(name_uniq, size, "%s_%u", name, jit->jit_dylib_count++); + } while(jit->lljit->getExecutionSession().getJITDylibByName(name_uniq)); + return name_uniq; + } + + static LLVMOrcJITDylibRef create_jit_dylib(const char * name) { + using llvm::orc::JITDylib; + LPJit* jit = get_instance(); + JITDylib& tmp = ExitOnErr(jit->lljit->createJITDylib(name)); + return wrap(&tmp); + } + + static void register_gallivm_state(gallivm_state *gallivm) { +#if DEBUG + LPJit* jit = get_instance(); + jit->gallivm_modules[gallivm->module_name] = gallivm; +#endif + } + + static void deregister_gallivm_state(gallivm_state *gallivm) { +#if DEBUG + LPJit* jit = get_instance(); + (void)jit->gallivm_modules.erase(gallivm->module_name); +#endif + } + + static void add_ir_module_to_jd( + LLVMOrcThreadSafeContextRef ts_context, + LLVMModuleRef mod, + LLVMOrcJITDylibRef jd) { + using llvm::Module; + using llvm::orc::ThreadSafeModule; + using llvm::orc::JITDylib; + ThreadSafeModule tsm( + std::unique_ptr<Module>(llvm::unwrap(mod)), *::unwrap(ts_context)); + ExitOnErr(get_instance()->lljit->addIRModule( + *::unwrap(jd), std::move(tsm) + )); + } + + static void add_mapping_to_jd( + LLVMValueRef sym, + void *addr, + LLVMOrcJITDylibRef jd) { + using llvm::JITEvaluatedSymbol; + using llvm::orc::ExecutionSession; + using llvm::orc::JITDylib; + using llvm::orc::SymbolMap; + JITDylib* JD = ::unwrap(jd); + auto& es = LPJit::get_instance()->lljit->getExecutionSession(); + auto name = es.intern(llvm::unwrap(sym)->getName()); + SymbolMap map(1); + map[name] = JITEvaluatedSymbol::fromPointer(addr); + auto munit = llvm::orc::absoluteSymbols(map); + llvm::cantFail(JD->define(std::move(munit))); + } + + static void *lookup_in_jd( + const char *func_name, + LLVMOrcJITDylibRef jd) { + using llvm::orc::JITDylib; + using llvm::JITEvaluatedSymbol; + using llvm::orc::ExecutorAddr; + JITDylib* JD = ::unwrap(jd); + auto func = ExitOnErr(LPJit::get_instance()->lljit->lookup(*JD, func_name)); +#if LLVM_VERSION_MAJOR >= 15 + return func.toPtr<void *>(); +#else + return (void *)(func.getAddress()); +#endif + } + + static void remove_jd(LLVMOrcJITDylibRef jd) { + using llvm::orc::ExecutionSession; + using llvm::orc::JITDylib; + auto& es = LPJit::get_instance()->lljit->getExecutionSession(); + ExitOnErr(es.removeJITDylib(* ::unwrap(jd))); + } + + LLVMTargetMachineRef tm; + +private: + LPJit(); + ~LPJit() = default; + LPJit(const LPJit&) = delete; + LPJit& operator=(const LPJit&) = delete; + + static void init_native_targets(); + llvm::orc::JITTargetMachineBuilder create_jtdb(); + + static void init_lpjit() { + jit = new LPJit; + } + static LPJit* jit; + + std::unique_ptr<llvm::orc::LLJIT> lljit; + /* avoid name conflict */ + unsigned jit_dylib_count; + +#if DEBUG + /* map from module name to gallivm_state */ + llvm::StringMap<gallivm_state *> gallivm_modules; +#endif +}; + +LPJit* LPJit::jit = NULL; + +LLVMErrorRef module_transform(void *Ctx, LLVMModuleRef mod) { + int64_t time_begin = 0; + if (::gallivm_debug & GALLIVM_DEBUG_PERF) + time_begin = os_time_get(); +#if GALLIVM_USE_NEW_PASS == 1 + char passes[1024]; + passes[0] = 0; + + /* + * there should be some way to combine these two pass runs but I'm not seeing it, + * at the time of writing. + */ + strcpy(passes, "default<O0>"); + + LLVMPassBuilderOptionsRef opts = LLVMCreatePassBuilderOptions(); + LLVMRunPasses(mod, passes, LPJit::get_instance()->tm, opts); + + if (!(gallivm_perf & GALLIVM_PERF_NO_OPT)) + strcpy(passes, "sroa,early-cse,simplifycfg,reassociate,mem2reg,constprop,instcombine,"); + else + strcpy(passes, "mem2reg"); + + LLVMRunPasses(mod, passes, LPJit::get_instance()->tm, opts); + LLVMDisposePassBuilderOptions(opts); + + return LLVMErrorSuccess; + +#else /* GALLIVM_USE_NEW_PASS */ + LLVMPassManagerRef passmgr = LLVMCreateFunctionPassManagerForModule(mod); + +#if GALLIVM_HAVE_CORO == 1 + LLVMPassManagerRef cgpassmgr = LLVMCreatePassManager(); +#endif + +#if GALLIVM_HAVE_CORO == 1 +#if LLVM_VERSION_MAJOR <= 8 && (defined(PIPE_ARCH_AARCH64) || defined (PIPE_ARCH_ARM) || defined(PIPE_ARCH_S390) || defined(PIPE_ARCH_MIPS64)) + LLVMAddArgumentPromotionPass(cgpassmgr); + LLVMAddFunctionAttrsPass(cgpassmgr); +#endif + LLVMAddCoroEarlyPass(cgpassmgr); + LLVMAddCoroSplitPass(cgpassmgr); + LLVMAddCoroElidePass(cgpassmgr); +#endif + + if ((gallivm_perf & GALLIVM_PERF_NO_OPT) == 0) { + /* + * TODO: Evaluate passes some more - keeping in mind + * both quality of generated code and compile times. + */ + /* + * NOTE: if you change this, don't forget to change the output + * with GALLIVM_DEBUG_DUMP_BC in gallivm_compile_module. + */ + LLVMAddScalarReplAggregatesPass(passmgr); + LLVMAddEarlyCSEPass(passmgr); + LLVMAddCFGSimplificationPass(passmgr); + /* + * FIXME: LICM is potentially quite useful. However, for some + * rather crazy shaders the compile time can reach _hours_ per shader, + * due to licm implying lcssa (since llvm 3.5), which can take forever. + * Even for sane shaders, the cost of licm is rather high (and not just + * due to lcssa, licm itself too), though mostly only in cases when it + * can actually move things, so having to disable it is a pity. + * LLVMAddLICMPass(passmgr); + */ + LLVMAddReassociatePass(passmgr); + LLVMAddPromoteMemoryToRegisterPass(passmgr); +#if LLVM_VERSION_MAJOR <= 11 + LLVMAddConstantPropagationPass(passmgr); +#else + LLVMAddInstructionSimplifyPass(passmgr); +#endif + LLVMAddInstructionCombiningPass(passmgr); + LLVMAddGVNPass(passmgr); + } + else { + /* We need at least this pass to prevent the backends to fail in + * unexpected ways. + */ + LLVMAddPromoteMemoryToRegisterPass(passmgr); + } +#if GALLIVM_HAVE_CORO == 1 + LLVMAddCoroCleanupPass(passmgr); + + LLVMRunPassManager(cgpassmgr, mod); +#endif + /* Run optimization passes */ + LLVMInitializeFunctionPassManager(passmgr); + LLVMValueRef func; + func = LLVMGetFirstFunction(mod); + while (func) { + if (0) { + debug_printf("optimizing func %s...\n", LLVMGetValueName(func)); + } + + /* Disable frame pointer omission on debug/profile builds */ + /* XXX: And workaround http://llvm.org/PR21435 */ +#if defined(DEBUG) || defined(PROFILE) || defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) + LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim", "true"); + LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim-non-leaf", "true"); +#endif + + LLVMRunFunctionPassManager(passmgr, func); + func = LLVMGetNextFunction(func); + } + LLVMFinalizeFunctionPassManager(passmgr); + if (gallivm_debug & GALLIVM_DEBUG_PERF) { + int64_t time_end = os_time_get(); + int time_msec = (int)((time_end - time_begin) / 1000); + + const char *module_name = get_module_name(mod); + debug_printf("optimizing module %s took %d msec\n", + module_name, time_msec); + } + +#if GALLIVM_HAVE_CORO == 1 + LLVMDisposePassManager(cgpassmgr); +#endif + LLVMDisposePassManager(passmgr); + return LLVMErrorSuccess; +#endif /* GALLIVM_USE_NEW_PASS */ +} + +LLVMErrorRef module_transform_wrapper( + void *Ctx, LLVMOrcThreadSafeModuleRef *ModInOut, + LLVMOrcMaterializationResponsibilityRef MR) { + return LLVMOrcThreadSafeModuleWithModuleDo(*ModInOut, *module_transform, Ctx); +} + +LPJit::LPJit() :jit_dylib_count(0) { + using namespace llvm::orc; +#ifdef DEBUG + ::gallivm_debug = debug_get_option_gallivm_debug(); +#endif + + gallivm_perf = debug_get_flags_option("GALLIVM_PERF", lp_bld_perf_flags, 0 ); + + init_native_targets(); + JITTargetMachineBuilder JTMB = create_jtdb(); + tm = wrap(ExitOnErr(JTMB.createTargetMachine()).release()); + + /* Create an LLJIT instance with an ObjectLinkingLayer (JITLINK) + * or RuntimeDyld as the base layer. + * intel & perf listeners are not supported by ObjectLinkingLayer yet + */ + lljit = ExitOnErr( + LLJITBuilder() + .setJITTargetMachineBuilder(std::move(JTMB)) +#ifdef USE_JITLINK + .setObjectLinkingLayerCreator( + [&](ExecutionSession &ES, const llvm::Triple &TT) { + return std::make_unique<ObjectLinkingLayer>( + ES, ExitOnErr(llvm::jitlink::InProcessMemoryManager::Create())); + }) +#else +#if LLVM_USE_INTEL_JITEVENTS + .RegisterJITEventListener( + llvm::JITEventListener::createIntelJITEventListener()) +#endif +#endif + .create()); + + LLVMOrcIRTransformLayerRef TL = wrap(&lljit->getIRTransformLayer()); + LLVMOrcIRTransformLayerSetTransform(TL, *module_transform_wrapper, NULL); +} + +void LPJit::init_native_targets() { + // If we have a native target, initialize it to ensure it is linked in and + // usable by the JIT. + llvm::InitializeNativeTarget(); + + llvm::InitializeNativeTargetAsmPrinter(); + + llvm::InitializeNativeTargetDisassembler(); +#if DEBUG + { + char *env_llc_options = getenv("GALLIVM_LLC_OPTIONS"); + if (env_llc_options) { + char *option; + char *options[64] = {(char *) "llc"}; // Warning without cast + int n; + for (n = 0, option = strtok(env_llc_options, " "); option; n++, option = strtok(NULL, " ")) { + options[n + 1] = option; + } + if (::gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + debug_printf("llc additional options (%d):\n", n); + for (int i = 1; i <= n; i++) + debug_printf("\t%s\n", options[i]); + debug_printf("\n"); + } + LLVMParseCommandLineOptions(n + 1, options, NULL); + } + } + + /* For simulating less capable machines */ + if (debug_get_bool_option("LP_FORCE_SSE2", false)) { + extern struct util_cpu_caps_t util_cpu_caps; + assert(util_cpu_caps.has_sse2); + util_cpu_caps.has_sse3 = 0; + util_cpu_caps.has_ssse3 = 0; + util_cpu_caps.has_sse4_1 = 0; + util_cpu_caps.has_sse4_2 = 0; + util_cpu_caps.has_avx = 0; + util_cpu_caps.has_avx2 = 0; + util_cpu_caps.has_f16c = 0; + util_cpu_caps.has_fma = 0; + } +#endif + + if (util_get_cpu_caps()->has_avx2 || util_get_cpu_caps()->has_avx) { + ::lp_native_vector_width = 256; + } else { + /* Leave it at 128, even when no SIMD extensions are available. + * Really needs to be a multiple of 128 so can fit 4 floats. + */ + ::lp_native_vector_width = 128; + } + + ::lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH", + lp_native_vector_width); + +#if LLVM_VERSION_MAJOR < 4 + if (::lp_native_vector_width <= 128) { + /* Hide AVX support, as often LLVM AVX intrinsics are only guarded by + * "util_get_cpu_caps()->has_avx" predicate, and lack the + * "lp_native_vector_width > 128" predicate. And also to ensure a more + * consistent behavior, allowing one to test SSE2 on AVX machines. + * XXX: should not play games with util_cpu_caps directly as it might + * get used for other things outside llvm too. + */ + util_get_cpu_caps()->has_avx = 0; + util_get_cpu_caps()->has_avx2 = 0; + util_get_cpu_caps()->has_f16c = 0; + util_get_cpu_caps()->has_fma = 0; + } +#endif + +#ifdef PIPE_ARCH_PPC_64 + /* Set the NJ bit in VSCR to 0 so denormalized values are handled as + * specified by IEEE standard (PowerISA 2.06 - Section 6.3). This guarantees + * that some rounding and half-float to float handling does not round + * incorrectly to 0. + * XXX: should eventually follow same logic on all platforms. + * Right now denorms get explicitly disabled (but elsewhere) for x86, + * whereas ppc64 explicitly enables them... + */ + if (util_get_cpu_caps()->has_altivec) { + unsigned short mask[] = { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF, 0xFFFE, 0xFFFF }; + __asm ( + "mfvscr %%v1\n" + "vand %0,%%v1,%0\n" + "mtvscr %0" + : + : "r" (*mask) + ); + } +#endif +} + +llvm::orc::JITTargetMachineBuilder LPJit::create_jtdb() { + using namespace llvm; + using orc::JITTargetMachineBuilder; + +#if defined(_WIN32) && LLVM_VERSION_MAJOR < 15 + /* + * JITLink works on Windows, but only through ELF object format. + * + * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has + * different strings for MinGW/MSVC, so better play it safe and be + * explicit. + */ +# ifdef _WIN64 + JITTargetMachineBuilder JTMB((Triple("x86_64-pc-win32-elf"))); +# else + JITTargetMachineBuilder JTMB((Triple("i686-pc-win32-elf"))); +# endif +#else + /* + * llvm::sys::getProcessTriple() is bogus. It returns the host LLVM was + * compiled on. Be careful when doing cross compilation + */ + JITTargetMachineBuilder JTMB((Triple(sys::getProcessTriple()))); +#endif + + TargetOptions options; + /** + * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and + * friends for configuring code generation options, like stack alignment. + */ +#if defined(PIPE_ARCH_X86) && LLVM_VERSION_MAJOR < 13 + options.StackAlignmentOverride = 4; +#endif + + JTMB.setOptions(options); + + std::vector<std::string> MAttrs; + +#if LLVM_VERSION_MAJOR >= 4 && (defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) || defined(PIPE_ARCH_ARM)) + /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm + * and llvm-3.7+ for x86, which allows us to enable/disable + * code generation based on the results of cpuid on these + * architectures. + */ + StringMap<bool> features; + sys::getHostCPUFeatures(features); + + for (StringMapIterator<bool> f = features.begin(); + f != features.end(); + ++f) { + MAttrs.push_back(((*f).second ? "+" : "-") + (*f).first().str()); + } +#elif defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) + /* + * We need to unset attributes because sometimes LLVM mistakenly assumes + * certain features are present given the processor name. + * + * https://bugs.freedesktop.org/show_bug.cgi?id=92214 + * http://llvm.org/PR25021 + * http://llvm.org/PR19429 + * http://llvm.org/PR16721 + */ + MAttrs.push_back(util_get_cpu_caps()->has_sse ? "+sse" : "-sse" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse2 ? "+sse2" : "-sse2" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse3 ? "+sse3" : "-sse3" ); + MAttrs.push_back(util_get_cpu_caps()->has_ssse3 ? "+ssse3" : "-ssse3" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse4_1 ? "+sse4.1" : "-sse4.1"); + MAttrs.push_back(util_get_cpu_caps()->has_sse4_2 ? "+sse4.2" : "-sse4.2"); + /* + * AVX feature is not automatically detected from CPUID by the X86 target + * yet, because the old (yet default) JIT engine is not capable of + * emitting the opcodes. On newer llvm versions it is and at least some + * versions (tested with 3.3) will emit avx opcodes without this anyway. + */ + MAttrs.push_back(util_get_cpu_caps()->has_avx ? "+avx" : "-avx"); + MAttrs.push_back(util_get_cpu_caps()->has_f16c ? "+f16c" : "-f16c"); + MAttrs.push_back(util_get_cpu_caps()->has_fma ? "+fma" : "-fma"); + MAttrs.push_back(util_get_cpu_caps()->has_avx2 ? "+avx2" : "-avx2"); + /* disable avx512 and all subvariants */ + MAttrs.push_back("-avx512cd"); + MAttrs.push_back("-avx512er"); + MAttrs.push_back("-avx512f"); + MAttrs.push_back("-avx512pf"); + MAttrs.push_back("-avx512bw"); + MAttrs.push_back("-avx512dq"); + MAttrs.push_back("-avx512vl"); +#endif +#if defined(PIPE_ARCH_ARM) + if (!util_get_cpu_caps()->has_neon) { + MAttrs.push_back("-neon"); + MAttrs.push_back("-crypto"); + MAttrs.push_back("-vfp2"); + } +#endif + +#if defined(PIPE_ARCH_PPC) + MAttrs.push_back(util_get_cpu_caps()->has_altivec ? "+altivec" : "-altivec"); +#if (LLVM_VERSION_MAJOR < 4) + /* + * Make sure VSX instructions are disabled + * See LLVM bugs: + * https://llvm.org/bugs/show_bug.cgi?id=25503#c7 (fixed in 3.8.1) + * https://llvm.org/bugs/show_bug.cgi?id=26775 (fixed in 3.8.1) + * https://llvm.org/bugs/show_bug.cgi?id=33531 (fixed in 4.0) + * https://llvm.org/bugs/show_bug.cgi?id=34647 (llc performance on certain unusual shader IR; intro'd in 4.0, pending as of 5.0) + */ + if (util_get_cpu_caps()->has_altivec) { + MAttrs.push_back("-vsx"); + } +#else + /* + * Bug 25503 is fixed, by the same fix that fixed + * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1). + * BZ 33531 actually comprises more than one bug, all of + * which are fixed in LLVM 4.0. + * + * With LLVM 4.0 or higher: + * Make sure VSX instructions are ENABLED (if supported), unless + * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0. + */ + if (util_get_cpu_caps()->has_altivec) { + MAttrs.push_back(util_get_cpu_caps()->has_vsx ? "+vsx" : "-vsx"); + } +#endif +#endif + +#if defined(PIPE_ARCH_MIPS64) + MAttrs.push_back(util_get_cpu_caps()->has_msa ? "+msa" : "-msa"); + /* MSA requires a 64-bit FPU register file */ + MAttrs.push_back("+fp64"); +#endif + + JTMB.addFeatures(MAttrs); + + if (::gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + int n = MAttrs.size(); + if (n > 0) { + debug_printf("llc -mattr option(s): "); + for (int i = 0; i < n; i++) + debug_printf("%s%s", MAttrs[i].c_str(), (i < n - 1) ? "," : ""); + debug_printf("\n"); + } + } + + std::string MCPU = llvm::sys::getHostCPUName().str(); + /* + * Note that the MAttrs set above will be sort of ignored (since we should + * not set any which would not be set by specifying the cpu anyway). + * It ought to be safe though since getHostCPUName() should include bits + * not only from the cpu but environment as well (for instance if it's safe + * to use avx instructions which need OS support). According to + * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this + * right it may be necessary to specify older cpu (or disable mattrs) though + * when not using MCJIT so no instructions are generated which the old JIT + * can't handle. Not entirely sure if we really need to do anything yet. + * + * Not sure if the above is also the case for ORCJIT, but we need set CPU + * manually since we don't use JITTargetMachineBuilder::detectHost() + */ + +#ifdef PIPE_ARCH_PPC_64 + /* + * Large programs, e.g. gnome-shell and firefox, may tax the addressability + * of the Medium code model once dynamically generated JIT-compiled shader + * programs are linked in and relocated. Yet the default code model as of + * LLVM 8 is Medium or even Small. + * The cost of changing from Medium to Large is negligible: + * - an additional 8-byte pointer stored immediately before the shader entrypoint; + * - change an add-immediate (addis) instruction to a load (ld). + */ + JTMB.setCodeModel(CodeModel::Large); + +#if UTIL_ARCH_LITTLE_ENDIAN + /* + * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL", + * resulting in (big-endian) "generic" being returned on + * little-endian Power8NVL systems. The result was that code that + * attempted to load the least significant 32 bits of a 64-bit quantity + * from memory loaded the wrong half. This resulted in failures in some + * Piglit tests, e.g. + * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint + */ + if (MCPU == "generic") + MCPU = "pwr8"; +#endif +#endif + +#if defined(PIPE_ARCH_MIPS64) + /* + * ls3a4000 CPU and ls2k1000 SoC is a mips64r5 compatible with MSA SIMD + * instruction set implemented, while ls3a3000 is mips64r2 compatible + * only. getHostCPUName() return "generic" on all loongson + * mips CPU currently. So we override the MCPU to mips64r5 if MSA is + * implemented, feedback to mips64r2 for all other ordinary mips64 cpu. + */ + if (MCPU == "generic") + MCPU = util_get_cpu_caps()->has_msa ? "mips64r5" : "mips64r2"; +#endif + + JTMB.setCPU(MCPU); + if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + debug_printf("llc -mcpu option: %s\n", MCPU.c_str()); + } + + return JTMB; +} + + +} /* Anonymous namespace */ + +unsigned +lp_build_init_native_width(void) +{ + // Default to 256 until we're confident llvmpipe with 512 is as correct and not slower than 256 + lp_native_vector_width = MIN2(util_get_cpu_caps()->max_vector_bits, 256); + assert(lp_native_vector_width); + + lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH", lp_native_vector_width); + assert(lp_native_vector_width); + + return lp_native_vector_width; +} + +bool +lp_build_init(void) +{ + (void)LPJit::get_instance(); + return true; +} + +bool +init_gallivm_state(struct gallivm_state *gallivm, const char *name, + LLVMOrcThreadSafeContextRef context, struct lp_cached_code *cache) +{ + assert(!gallivm->context); + assert(!gallivm->_ts_context); + assert(!gallivm->module); + + if (!lp_build_init()) + return false; + + // cache is not implemented + gallivm->cache = cache; + if (gallivm->cache) + gallivm->cache->data_size = 0; + + gallivm->_ts_context = context; + gallivm->context = LLVMOrcThreadSafeContextGetContext(context); + + gallivm->module_name = LPJit::get_unique_name(name); + gallivm->module = LLVMModuleCreateWithNameInContext(gallivm->module_name, + gallivm->context); +#if defined(PIPE_ARCH_X86) + lp_set_module_stack_alignment_override(gallivm->module, 4); +#endif + gallivm->builder = LLVMCreateBuilderInContext(gallivm->context); + gallivm->_per_module_jd = LPJit::create_jit_dylib(gallivm->module_name); + + gallivm->target = LLVMCreateTargetDataLayout(LPJit::get_instance()->tm); + + return true; +} + +struct gallivm_state * +gallivm_create(const char *name, LLVMOrcThreadSafeContextRef context, + struct lp_cached_code *cache){ + struct gallivm_state *gallivm; + + gallivm = CALLOC_STRUCT(gallivm_state); + if (gallivm) { + if (!init_gallivm_state(gallivm, name, context, cache)) { + FREE(gallivm); + gallivm = NULL; + } + } + + assert(gallivm != NULL); + return gallivm; +} + +void +gallivm_destroy(struct gallivm_state *gallivm) +{ + LPJit::remove_jd(gallivm->_per_module_jd); + gallivm->_per_module_jd = nullptr; +} + +void +gallivm_free_ir(struct gallivm_state *gallivm) +{ + if (gallivm->module) + LLVMDisposeModule(gallivm->module); + FREE(gallivm->module_name); + + if (gallivm->target) { + LLVMDisposeTargetData(gallivm->target); + } + + if (gallivm->builder) + LLVMDisposeBuilder(gallivm->builder); + + gallivm->target = NULL; + gallivm->module=NULL; + gallivm->module_name=NULL; + gallivm->builder=NULL; + gallivm->context=NULL; + gallivm->_ts_context=NULL; + gallivm->cache=NULL; + LPJit::deregister_gallivm_state(gallivm); +} + +void +gallivm_verify_function(struct gallivm_state *gallivm, + LLVMValueRef func) +{ + /* Verify the LLVM IR. If invalid, dump and abort */ +#ifdef DEBUG + if (LLVMVerifyFunction(func, LLVMPrintMessageAction)) { + lp_debug_dump_value(func); + assert(0); + return; + } +#endif + + if (gallivm_debug & GALLIVM_DEBUG_IR) { + /* Print the LLVM IR to stderr */ + lp_debug_dump_value(func); + debug_printf("\n"); + } +return; +} + +void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr) +{ + LPJit::add_mapping_to_jd(sym, addr, gallivm->_per_module_jd); +} + +void lp_init_clock_hook(struct gallivm_state *gallivm) +{ + if (gallivm->get_time_hook) + return; + + LLVMTypeRef get_time_type = LLVMFunctionType(LLVMInt64TypeInContext(gallivm->context), NULL, 0, 1); + gallivm->get_time_hook = LLVMAddFunction(gallivm->module, "get_time_hook", get_time_type); +} + +void +gallivm_compile_module(struct gallivm_state *gallivm) +{ + if (gallivm->debug_printf_hook) + gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook, + (void *)debug_printf); + + LPJit::add_ir_module_to_jd(gallivm->_ts_context, gallivm->module, + gallivm->_per_module_jd); + /* ownership of module is now transferred into orc jit, + * disallow modifying it + */ + LPJit::register_gallivm_state(gallivm); + gallivm->module=nullptr; + + /* defer compilation till first lookup by gallivm_jit_function */ +} + +func_pointer +gallivm_jit_function(struct gallivm_state *gallivm, + const char *func_name) +{ + return pointer_to_func( + LPJit::lookup_in_jd(func_name, gallivm->_per_module_jd)); +} + +unsigned +gallivm_get_perf_flags(void){ + return gallivm_perf; +} + +void +lp_set_module_stack_alignment_override(LLVMModuleRef MRef, unsigned align) +{ +#if LLVM_VERSION_MAJOR >= 13 + llvm::Module *M = llvm::unwrap(MRef); + M->setOverrideStackAlignment(align); +#endif +} diff --git a/src/gallium/auxiliary/meson.build b/src/gallium/auxiliary/meson.build index 4be71a4..c3fea24 100644 --- a/src/gallium/auxiliary/meson.build +++ b/src/gallium/auxiliary/meson.build @@ -343,7 +343,7 @@ if draw_with_llvm 'gallivm/lp_bld_gather.c', 'gallivm/lp_bld_gather.h', 'gallivm/lp_bld.h', - 'gallivm/lp_bld_init.c', + 'gallivm/lp_bld_init_orc.cpp', 'gallivm/lp_bld_init.h', 'gallivm/lp_bld_intr.c', 'gallivm/lp_bld_intr.h', diff --git a/src/gallium/drivers/llvmpipe/lp_context.c b/src/gallium/drivers/llvmpipe/lp_context.c index 8e5e8ce..cbb442b 100644 --- a/src/gallium/drivers/llvmpipe/lp_context.c +++ b/src/gallium/drivers/llvmpipe/lp_context.c @@ -49,6 +49,10 @@ #include "lp_screen.h" #include "lp_fence.h" +#if ! (GALLIVM_USE_ORCJIT == 1) +#define USE_GLOBAL_LLVM_CONTEXT +#endif + static void llvmpipe_destroy(struct pipe_context *pipe) { @@ -108,7 +112,11 @@ llvmpipe_destroy(struct pipe_context *pipe) llvmpipe_sampler_matrix_destroy(llvmpipe); #ifndef USE_GLOBAL_LLVM_CONTEXT +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(llvmpipe->context); +#else LLVMContextDispose(llvmpipe->context); +#endif #endif llvmpipe->context = NULL; @@ -258,15 +266,25 @@ llvmpipe_create_context(struct pipe_screen *screen, void *priv, #ifdef USE_GLOBAL_LLVM_CONTEXT llvmpipe->context = LLVMGetGlobalContext(); +#else +#if GALLIVM_USE_ORCJIT == 1 + llvmpipe->context = LLVMOrcCreateNewThreadSafeContext(); #else llvmpipe->context = LLVMContextCreate(); +#endif #endif if (!llvmpipe->context) goto fail; -#if LLVM_VERSION_MAJOR == 15 +#if GALLIVM_USE_ORCJIT == 1 +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(llvmpipe->context), false); +#endif +#else +#if LLVM_VERSION_MAJOR >= 15 LLVMContextSetOpaquePointers(llvmpipe->context, false); +#endif #endif /* diff --git a/src/gallium/drivers/llvmpipe/lp_context.h b/src/gallium/drivers/llvmpipe/lp_context.h index 7dd6a3f..d30aa89 100644 --- a/src/gallium/drivers/llvmpipe/lp_context.h +++ b/src/gallium/drivers/llvmpipe/lp_context.h @@ -190,7 +190,11 @@ struct llvmpipe_context { unsigned render_cond_offset; /** The LLVMContext to use for LLVM related work */ +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif int max_global_buffers; struct pipe_resource **global_buffers; diff --git a/src/gallium/drivers/llvmpipe/lp_screen.c b/src/gallium/drivers/llvmpipe/lp_screen.c index bd6379d..91bd065 100644 --- a/src/gallium/drivers/llvmpipe/lp_screen.c +++ b/src/gallium/drivers/llvmpipe/lp_screen.c @@ -942,7 +942,11 @@ static void lp_disk_cache_create(struct llvmpipe_screen *screen) { struct mesa_sha1 ctx; +#if GALLIVM_USE_ORCJIT == 1 + unsigned gallivm_perf = 0; +#else unsigned gallivm_perf = gallivm_get_perf_flags(); +#endif unsigned char sha1[20]; char cache_id[20 * 2 + 1]; _mesa_sha1_init(&ctx); diff --git a/src/gallium/drivers/llvmpipe/lp_state_cs.c b/src/gallium/drivers/llvmpipe/lp_state_cs.c index 97c094c..f7f4cbd 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_cs.c +++ b/src/gallium/drivers/llvmpipe/lp_state_cs.c @@ -404,6 +404,10 @@ generate_compute(struct llvmpipe_context *lp, lp_build_coro_add_presplit(coro); variant->function = function; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif for (i = 0; i < CS_ARG_MAX - !is_mesh; ++i) { if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) { @@ -957,6 +961,10 @@ llvmpipe_remove_cs_shader_variant(struct llvmpipe_context *lp, lp->nr_cs_variants--; lp->nr_cs_instrs -= variant->nr_instrs; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -1222,12 +1230,23 @@ generate_variant(struct llvmpipe_context *lp, generate_compute(lp, shader, variant); +#if GALLIVM_USE_ORCJIT == 1 +/* module has been moved into ORCJIT after gallivm_compile_module */ + lp_build_coro_add_malloc_hooks(variant->gallivm); + variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); + + gallivm_compile_module(variant->gallivm); + + variant->jit_function = (lp_jit_cs_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else gallivm_compile_module(variant->gallivm); variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); variant->jit_function = (lp_jit_cs_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) { lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key); diff --git a/src/gallium/drivers/llvmpipe/lp_state_cs.h b/src/gallium/drivers/llvmpipe/lp_state_cs.h index 855f195..26b6e87 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_cs.h +++ b/src/gallium/drivers/llvmpipe/lp_state_cs.h @@ -91,6 +91,9 @@ struct lp_compute_shader_variant LLVMTypeRef jit_vertex_header_ptr_type; LLVMTypeRef jit_prim_type; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char* function_name; +#endif lp_jit_cs_func jit_function; /* Total number of LLVM instructions generated */ diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs.c b/src/gallium/drivers/llvmpipe/lp_state_fs.c index 78c3b25..4e0b693 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_fs.c +++ b/src/gallium/drivers/llvmpipe/lp_state_fs.c @@ -3217,6 +3217,10 @@ generate_fragment(struct llvmpipe_context *lp, LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->function[partial_mask] = function; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name[partial_mask] = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name[partial_mask], func_name); +#endif /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? @@ -3926,20 +3930,37 @@ generate_variant(struct llvmpipe_context *lp, * Compile everything */ +#if GALLIVM_USE_ORCJIT == 1 +/* module has been moved into ORCJIT after gallivm_compile_module */ + variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); + + gallivm_compile_module(variant->gallivm); +#else gallivm_compile_module(variant->gallivm); variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); +#endif if (variant->function[RAST_EDGE_TEST]) { variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func) +#if GALLIVM_USE_ORCJIT == 1 + gallivm_jit_function(variant->gallivm, + variant->function_name[RAST_EDGE_TEST]); +#else gallivm_jit_function(variant->gallivm, variant->function[RAST_EDGE_TEST]); +#endif } if (variant->function[RAST_WHOLE]) { variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) +#if GALLIVM_USE_ORCJIT == 1 + gallivm_jit_function(variant->gallivm, + variant->function_name[RAST_WHOLE]); +#else gallivm_jit_function(variant->gallivm, variant->function[RAST_WHOLE]); +#endif } else if (!variant->jit_function[RAST_WHOLE]) { variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) variant->jit_function[RAST_EDGE_TEST]; @@ -3948,7 +3969,11 @@ generate_variant(struct llvmpipe_context *lp, if (linear_pipeline) { if (variant->linear_function) { variant->jit_linear_llvm = (lp_jit_linear_llvm_func) +#if GALLIVM_USE_ORCJIT == 1 + gallivm_jit_function(variant->gallivm, variant->linear_function_name); +#else gallivm_jit_function(variant->gallivm, variant->linear_function); +#endif } /* @@ -4136,6 +4161,14 @@ llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp, { gallivm_destroy(variant->gallivm); lp_fs_reference(lp, &variant->shader, NULL); +#if GALLIVM_USE_ORCJIT == 1 + if (variant->function_name[RAST_EDGE_TEST]) + FREE(variant->function_name[RAST_EDGE_TEST]); + if (variant->function_name[RAST_WHOLE]) + FREE(variant->function_name[RAST_WHOLE]); + if (variant->linear_function_name) + FREE(variant->linear_function_name); +#endif FREE(variant); } diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs.h b/src/gallium/drivers/llvmpipe/lp_state_fs.h index 023b7e8..fff6ebe 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_fs.h +++ b/src/gallium/drivers/llvmpipe/lp_state_fs.h @@ -169,6 +169,9 @@ struct lp_fragment_shader_variant LLVMTypeRef jit_linear_textures_type; LLVMValueRef function[2]; // [RAST_WHOLE], [RAST_EDGE_TEST] +#if GALLIVM_USE_ORCJIT == 1 + char* function_name[2]; +#endif lp_jit_frag_func jit_function[2]; // [RAST_WHOLE], [RAST_EDGE_TEST] @@ -178,6 +181,9 @@ struct lp_fragment_shader_variant /* Functions within the linear path: */ LLVMValueRef linear_function; +#if GALLIVM_USE_ORCJIT == 1 + char* linear_function_name; +#endif lp_jit_linear_llvm_func jit_linear_llvm; /* Bitmask to say what cbufs are unswizzled */ diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c b/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c index 8d94691..54d8554 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c +++ b/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c @@ -298,6 +298,10 @@ llvmpipe_fs_variant_linear_llvm(struct llvmpipe_context *lp, LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->linear_function = function; +#if GALLIVM_USE_ORCJIT == 1 + variant->linear_function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->linear_function_name, func_name); +#endif /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? diff --git a/src/gallium/drivers/llvmpipe/lp_state_setup.c b/src/gallium/drivers/llvmpipe/lp_state_setup.c index a4900e6..99afcd5 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_setup.c +++ b/src/gallium/drivers/llvmpipe/lp_state_setup.c @@ -687,6 +687,10 @@ generate_setup_variant(struct lp_setup_variant_key *key, arg_types, ARRAY_SIZE(arg_types), 0); variant->function = LLVMAddFunction(gallivm->module, func_name, func_type); +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif if (!variant->function) goto fail; @@ -730,8 +734,13 @@ generate_setup_variant(struct lp_setup_variant_key *key, gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_function = (lp_jit_setup_triangle) + gallivm_jit_function(gallivm, variant->function_name); +#else variant->jit_function = (lp_jit_setup_triangle) gallivm_jit_function(gallivm, variant->function); +#endif if (!variant->jit_function) goto fail; diff --git a/src/gallium/drivers/llvmpipe/lp_state_setup.h b/src/gallium/drivers/llvmpipe/lp_state_setup.h index ef20893..1e9c6db 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_setup.h +++ b/src/gallium/drivers/llvmpipe/lp_state_setup.h @@ -66,6 +66,9 @@ struct lp_setup_variant { * assembly. */ LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif /* The actual generated setup function: */ diff --git a/src/gallium/drivers/llvmpipe/lp_test_arit.c b/src/gallium/drivers/llvmpipe/lp_test_arit.c index fd8489f..fe192f5 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_arit.c +++ b/src/gallium/drivers/llvmpipe/lp_test_arit.c @@ -417,7 +417,11 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned { char test_name[128]; snprintf(test_name, sizeof test_name, "%s.v%u", test->name, length); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef test_func; unary_func_t test_func_jit; @@ -433,9 +437,16 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned in[i] = 1.0; } +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -443,7 +454,11 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_name); +#else test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_func); +#endif gallivm_free_ir(gallivm); @@ -512,7 +527,11 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif align_free(in); align_free(out); diff --git a/src/gallium/drivers/llvmpipe/lp_test_blend.c b/src/gallium/drivers/llvmpipe/lp_test_blend.c index 7851bf2..f066a68 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_blend.c +++ b/src/gallium/drivers/llvmpipe/lp_test_blend.c @@ -131,16 +131,24 @@ dump_blend_type(FILE *fp, } +#if GALLIVM_USE_ORCJIT == 1 +static const char * +add_blend_test(struct gallivm_state *gallivm, + const struct pipe_blend_state *blend, + struct lp_type type) +#else static LLVMValueRef add_blend_test(struct gallivm_state *gallivm, const struct pipe_blend_state *blend, struct lp_type type) +#endif { LLVMModuleRef module = gallivm->module; LLVMContextRef context = gallivm->context; LLVMTypeRef vec_type; LLVMTypeRef args[5]; LLVMValueRef func; + const char *func_name = "test"; LLVMValueRef src_ptr; LLVMValueRef src1_ptr; LLVMValueRef dst_ptr; @@ -160,7 +168,7 @@ add_blend_test(struct gallivm_state *gallivm, vec_type = lp_build_vec_type(gallivm, type); args[4] = args[3] = args[2] = args[1] = args[0] = LLVMPointerType(vec_type, 0); - func = LLVMAddFunction(module, "test", LLVMFunctionType(LLVMVoidTypeInContext(context), args, 5, 0)); + func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(context), args, 5, 0)); LLVMSetFunctionCallConv(func, LLVMCCallConv); src_ptr = LLVMGetParam(func, 0); src1_ptr = LLVMGetParam(func, 1); @@ -188,7 +196,11 @@ add_blend_test(struct gallivm_state *gallivm, gallivm_verify_function(gallivm, func); +#if GALLIVM_USE_ORCJIT == 1 + return func_name; +#else return func; +#endif } @@ -437,9 +449,17 @@ test_one(unsigned verbose, const struct pipe_blend_state *blend, struct lp_type type) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; +#if GALLIVM_USE_ORCJIT == 1 + const char *func; +#else LLVMValueRef func = NULL; +#endif blend_test_ptr_t blend_test_ptr; bool success; const unsigned n = LP_TEST_NUM_SAMPLES; @@ -451,9 +471,16 @@ test_one(unsigned verbose, if (verbose >= 1) dump_blend_type(stdout, blend, type); +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -584,7 +611,11 @@ test_one(unsigned verbose, write_tsv_row(fp, blend, type, cycles_avg, success); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_test_conv.c b/src/gallium/drivers/llvmpipe/lp_test_conv.c index 9fb6448..ac14185 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_conv.c +++ b/src/gallium/drivers/llvmpipe/lp_test_conv.c @@ -96,16 +96,24 @@ dump_conv_types(FILE *fp, } +#if GALLIVM_USE_ORCJIT == 1 +static const char * +add_conv_test(struct gallivm_state *gallivm, + struct lp_type src_type, unsigned num_srcs, + struct lp_type dst_type, unsigned num_dsts) +#else static LLVMValueRef add_conv_test(struct gallivm_state *gallivm, struct lp_type src_type, unsigned num_srcs, struct lp_type dst_type, unsigned num_dsts) +#endif { LLVMModuleRef module = gallivm->module; LLVMContextRef context = gallivm->context; LLVMBuilderRef builder = gallivm->builder; LLVMTypeRef args[2]; LLVMValueRef func; + const char *func_name = "test"; LLVMValueRef src_ptr; LLVMValueRef dst_ptr; LLVMBasicBlockRef block; @@ -118,7 +126,7 @@ add_conv_test(struct gallivm_state *gallivm, args[0] = LLVMPointerType(src_vec_type, 0); args[1] = LLVMPointerType(dst_vec_type, 0); - func = LLVMAddFunction(module, "test", + func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(context), args, 2, 0)); LLVMSetFunctionCallConv(func, LLVMCCallConv); @@ -146,7 +154,11 @@ add_conv_test(struct gallivm_state *gallivm, gallivm_verify_function(gallivm, func); +#if GALLIVM_USE_ORCJIT == 1 + return func_name; +#else return func; +#endif } @@ -157,9 +169,17 @@ test_one(unsigned verbose, struct lp_type src_type, struct lp_type dst_type) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; +#if GALLIVM_USE_ORCJIT == 1 + const char *func; +#else LLVMValueRef func = NULL; +#endif conv_test_ptr_t conv_test_ptr; bool success; const unsigned n = LP_TEST_NUM_SAMPLES; @@ -222,9 +242,16 @@ test_one(unsigned verbose, eps *= 2; } +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -337,7 +364,11 @@ test_one(unsigned verbose, write_tsv_row(fp, src_type, dst_type, cycles_avg, success); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_test_format.c b/src/gallium/drivers/llvmpipe/lp_test_format.c index 6d3a60b..81d0753 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_format.c +++ b/src/gallium/drivers/llvmpipe/lp_test_format.c @@ -79,9 +79,9 @@ static LLVMValueRef add_fetch_rgba_test(struct gallivm_state *gallivm, unsigned verbose, const struct util_format_description *desc, struct lp_type type, - unsigned use_cache) + unsigned use_cache, + char *name) { - char name[256]; LLVMContextRef context = gallivm->context; LLVMModuleRef module = gallivm->module; LLVMBuilderRef builder = gallivm->builder; @@ -96,7 +96,7 @@ add_fetch_rgba_test(struct gallivm_state *gallivm, unsigned verbose, LLVMValueRef rgba; LLVMValueRef cache = NULL; - snprintf(name, sizeof name, "fetch_%s_%s", desc->short_name, + snprintf(name, 64 * sizeof(char), "fetch_%s_%s", desc->short_name, type.floating ? "float" : "unorm8"); args[0] = LLVMPointerType(lp_build_vec_type(gallivm, type), 0); @@ -139,9 +139,14 @@ test_format_float(unsigned verbose, FILE *fp, const struct util_format_description *desc, unsigned use_cache) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef fetch = NULL; + char fetch_name[64]; fetch_ptr_t fetch_ptr; alignas(16) uint8_t packed[UTIL_FORMAT_MAX_PACKED_BYTES]; alignas(16) float unpacked[4]; @@ -149,18 +154,29 @@ test_format_float(unsigned verbose, FILE *fp, bool success = true; unsigned i, j, k, l; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module_float", context, NULL); fetch = add_fetch_rgba_test(gallivm, verbose, desc, - lp_float32_vec4_type(), use_cache); + lp_float32_vec4_type(), use_cache, fetch_name); gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch_name); +#else fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch); +#endif gallivm_free_ir(gallivm); @@ -228,7 +244,11 @@ test_format_float(unsigned verbose, FILE *fp, } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif if (fp) write_tsv_row(fp, desc, success); @@ -243,9 +263,14 @@ test_format_unorm8(unsigned verbose, FILE *fp, const struct util_format_description *desc, unsigned use_cache) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef fetch = NULL; + char fetch_name[64]; fetch_ptr_t fetch_ptr; alignas(16) uint8_t packed[UTIL_FORMAT_MAX_PACKED_BYTES]; uint8_t unpacked[4]; @@ -253,18 +278,29 @@ test_format_unorm8(unsigned verbose, FILE *fp, bool success = true; unsigned i, j, k, l; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module_unorm8", context, NULL); fetch = add_fetch_rgba_test(gallivm, verbose, desc, - lp_unorm8_vec4_type(), use_cache); + lp_unorm8_vec4_type(), use_cache, fetch_name); gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch_name); +#else fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch); +#endif gallivm_free_ir(gallivm); @@ -331,7 +367,11 @@ test_format_unorm8(unsigned verbose, FILE *fp, } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif if (fp) write_tsv_row(fp, desc, success); diff --git a/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c b/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c new file mode 100644 index 0000000..2acb339 --- /dev/null +++ b/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c @@ -0,0 +1,172 @@ +/************************************************************************** + * + * Copyright 2010 VMware, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + + +#include <stdlib.h> +#include <stdio.h> + +#include "util/u_pointer.h" +#include "gallivm/lp_bld.h" +#include "gallivm/lp_bld_init.h" +#include "gallivm/lp_bld_assert.h" +#include "gallivm/lp_bld_printf.h" + +#include "lp_test.h" + + +struct printf_test_case { + int foo; +}; + +void +write_tsv_header(FILE *fp) +{ + fprintf(fp, + "result\t" + "format\n"); + + fflush(fp); +} + + + +typedef void (*test_printf_t)(int i); + + +static LLVMValueRef +add_printf_test(struct gallivm_state *gallivm, int n, char *func_name) +{ + LLVMModuleRef module = gallivm->module; + LLVMTypeRef args[1] = { LLVMIntTypeInContext(gallivm->context, 32) }; + snprintf(func_name, 64 * sizeof(char), "test_lookup_multiple_%d", n); + LLVMValueRef func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); + LLVMBuilderRef builder = gallivm->builder; + LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, func, "entry"); + + LLVMSetFunctionCallConv(func, LLVMCCallConv); + + LLVMPositionBuilderAtEnd(builder, block); + lp_build_printf(gallivm, "hello, world from "); + lp_build_printf(gallivm, func_name); + lp_build_printf(gallivm, "print 5 6: %d %d\n", LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 5, 0), + LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 6, 0)); + + /* Also test lp_build_assert(). This should not fail. */ + lp_build_assert(gallivm, LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 1, 0), "assert(1)"); + + LLVMBuildRetVoid(builder); + + gallivm_verify_function(gallivm, func); + + return func; +} + + +static bool +test_lookup_multiple(unsigned verbose, FILE *fp, + const struct printf_test_case *testcase) +{ + struct gallivm_state *gallivm; + const int N = 10; + LLVMValueRef func[N]; + char func_name[N][64]; + test_printf_t test_lookup_multiple_func[N]; + bool success = true; + int i; + +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else + LLVMContextRef context = LLVMContextCreate(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(context, false); +#endif +#endif + gallivm = gallivm_create("test_module", context, NULL); + + for(i = 0; i < N; i++){ + func[i] = add_printf_test(gallivm, i, func_name[i]); + } + + gallivm_compile_module(gallivm); + +#if GALLIVM_USE_ORCJIT == 1 + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i] = (test_printf_t) gallivm_jit_function(gallivm, func_name[i]); + } + (void)func; +#else + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i] = (test_printf_t) gallivm_jit_function(gallivm, func[i]); + } +#endif + + gallivm_free_ir(gallivm); + + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i](0); + } + + gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else + LLVMContextDispose(context); +#endif + + return success; +} + + +bool +test_all(unsigned verbose, FILE *fp) +{ + bool success = true; + + test_lookup_multiple(verbose, fp, NULL); + + return success; +} + + +bool +test_some(unsigned verbose, FILE *fp, + unsigned long n) +{ + return test_all(verbose, fp); +} + + +bool +test_single(unsigned verbose, FILE *fp) +{ + printf("no test_single()"); + return true; +} diff --git a/src/gallium/drivers/llvmpipe/lp_test_printf.c b/src/gallium/drivers/llvmpipe/lp_test_printf.c index 7e53aa5..fb2a684 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_printf.c +++ b/src/gallium/drivers/llvmpipe/lp_test_printf.c @@ -57,12 +57,18 @@ write_tsv_header(FILE *fp) typedef void (*test_printf_t)(int i); +#if GALLIVM_USE_ORCJIT == 1 +static const char * +add_printf_test(struct gallivm_state *gallivm) +#else static LLVMValueRef add_printf_test(struct gallivm_state *gallivm) +#endif { LLVMModuleRef module = gallivm->module; + const char *func_name = "test_printf"; LLVMTypeRef args[1] = { LLVMIntTypeInContext(gallivm->context, 32) }; - LLVMValueRef func = LLVMAddFunction(module, "test_printf", LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); + LLVMValueRef func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); LLVMBuilderRef builder = gallivm->builder; LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, func, "entry"); @@ -80,7 +86,11 @@ add_printf_test(struct gallivm_state *gallivm) gallivm_verify_function(gallivm, func); +#if GALLIVM_USE_ORCJIT == 1 + return func_name; +#else return func; +#endif } @@ -89,15 +99,30 @@ static bool test_printf(unsigned verbose, FILE *fp, const struct printf_test_case *testcase) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; +#if GALLIVM_USE_ORCJIT == 1 + const char *test; +#else LLVMValueRef test; +#endif test_printf_t test_printf_func; bool success = true; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR >= 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -112,7 +137,11 @@ test_printf(unsigned verbose, FILE *fp, test_printf_func(0); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_texture_handle.c b/src/gallium/drivers/llvmpipe/lp_texture_handle.c index 9bb8f22..bfd1134 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture_handle.c +++ b/src/gallium/drivers/llvmpipe/lp_texture_handle.c @@ -200,11 +200,19 @@ llvmpipe_sampler_matrix_destroy(struct llvmpipe_context *ctx) util_dynarray_fini(&ctx->sampler_matrix.gallivms); } +#if GALLIVM_USE_ORCJIT == 1 +static void * +compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function_ref, const char *function, + uint8_t cache_key[SHA1_DIGEST_LENGTH]) +{ + gallivm_verify_function(gallivm, function_ref); +#else static void * compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function, uint8_t cache_key[SHA1_DIGEST_LENGTH]) { gallivm_verify_function(gallivm, function); +#endif gallivm_compile_module(gallivm); void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function)); @@ -333,7 +341,11 @@ compile_image_function(struct llvmpipe_context *ctx, struct lp_static_texture_st free(image_soa); +#if GALLIVM_USE_ORCJIT == 1 + return compile_function(ctx, gallivm, function, "image", cache_key); +#else return compile_function(ctx, gallivm, function, cache_key); +#endif } static void * @@ -471,7 +483,11 @@ compile_sample_function(struct llvmpipe_context *ctx, struct lp_static_texture_s free(sampler_soa); +#if GALLIVM_USE_ORCJIT == 1 + return compile_function(ctx, gallivm, function, "sample", cache_key); +#else return compile_function(ctx, gallivm, function, cache_key); +#endif } static void * @@ -551,7 +567,11 @@ compile_size_function(struct llvmpipe_context *ctx, struct lp_static_texture_sta free(sampler_soa); +#if GALLIVM_USE_ORCJIT == 1 + return compile_function(ctx, gallivm, function, "size", cache_key); +#else return compile_function(ctx, gallivm, function, cache_key); +#endif } static void diff --git a/src/gallium/drivers/llvmpipe/meson.build b/src/gallium/drivers/llvmpipe/meson.build index 38ff889..d405c67 100644 --- a/src/gallium/drivers/llvmpipe/meson.build +++ b/src/gallium/drivers/llvmpipe/meson.build @@ -132,7 +132,7 @@ driver_swrast = declare_dependency( if with_tests and with_gallium_softpipe and draw_with_llvm foreach t : ['lp_test_format', 'lp_test_arit', 'lp_test_blend', - 'lp_test_conv', 'lp_test_printf'] + 'lp_test_conv', 'lp_test_printf', 'lp_test_lookup_multiple'] test( t, executable(
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.
浙ICP备2022010568号-2