/jVMxddlmZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddlm Z ddl ZddlmZej$rddlmZd%dZGdd e Zd&d Zd Zd Zd'd Zd(dZd)dZGddZ d*dZGddZGddeZ d+dZ!GddZ"GddZ#GddZ$dZ%d,dZ&e&f d-dZ'd.d/d Z(Gd!d"Z)Gd#d$Z*ee+e$ejXdzgejXdzfZ-ee+ejHejXdzgejXdzfZ.y)0) annotationsN)CallableSequence)IntEnum)Any) _pybind_statec|dk(rtjjS|dk(rtjjS|dk(rtjj S|dk(rtjj S|dk(rtjj S|dk(rtjjS|dk(rtjjStd|z) NcudacanncpudmlwebgpugpunpuzUnsupported device type: ) C OrtDevicer r r r rrr Exception device_types t/root/repos/pocs/video-evals/.venv/lib/python3.12/site-packages/onnxruntime/capi/onnxruntime_inference_collection.pyget_ort_device_typersf{{!!  {{!!  {{   {{   {{!!##  {{   {{  3kABBc0eZdZdZdZdZdZdZdZdZ dZ d Z y ) OrtDeviceVendorIdzO>OSa>a  % % u(*;*;*M*M*OSaSsSsSu*uurc:t||std|dy)zReject an OrtValue that the target session cannot share buffers with. ``action`` is the verb used in the message, e.g. ``"used with"`` or ``"bound to"``. z Session-scoped OrtValue must be z the session that created it.N)r<r2)r:r;actions r(_validate_ortvalue_session_compatibilityr?fs* +8^ D;F8C`abb ErcleZdZdZd ddZeddZddZddZddZ ddZ dd Z dd Z dd Z dd Zy) AdapterFormatzK This class is used to create adapter files from python structures NcJ|tj|_y||_yN)rrA_adapter)selfadapters r__init__zAdapterFormat.__init__ts ?OO-DM#DMrcRttjj|SrC)rAr read_adapter) file_paths rrIzAdapterFormat.read_adapterzsQ__99)DEErc:|jj|y)z This function writes a file at the specified location in onnxrunitme adapter format containing Lora parameters. :param file_path: absolute path for the adapter N)rDexport_adapter)rErJs rrLzAdapterFormat.export_adapter~s $$Y/rc.|jjSrC)rDformat_versionrEs rget_format_versionz AdapterFormat.get_format_versions}}+++rc&||j_yrCrDadapter_version)rErSs rset_adapter_versionz!AdapterFormat.set_adapter_versions(7 %rc.|jjSrCrRrOs rget_adapter_versionz!AdapterFormat.get_adapter_versions}},,,rc&||j_yrCrD model_version)rErYs rset_model_versionzAdapterFormat.set_model_versions&3 #rc.|jjSrCrXrOs rget_model_versionzAdapterFormat.get_model_versions}}***rc|jDcic]\}}||jc}}|j_ycc}}w)z"Set adapter parameters for export.N)items _ortvaluerD parameters)rEparamskvs rset_parameterszAdapterFormat.set_parameterss.?E||~#Ntq!Aq{{N#N  #Ns?c|jjjDcic]\}}|t|c}}Scc}}w)zGet adapter parameters as a dict of name -> OrtValue. On read instances, the returned OrtValues are zero-copy views; the backing memory stays alive as long as any returned OrtValue is referenced. )rDr`r^OrtValue)rErbrcs rget_parameterszAdapterFormat.get_parameterss7 ,0==+C+C+I+I+KL41a8A;LLLsArCreturnNone)rJ os.PathLikerirA)rJrkrir1)rSr1rirj)rYr1rirj)radict[str, OrtValue]rirj)rirm)rrrrrG staticmethodrIrLrPrTrVrZr\rdrgr'rrrArAosL$ FF0,8-4+OMrrAc`|ggfStjfd}t|tjjs t d|t|tjjs t dt |t |k7r t dtd|Ds t dtd|Ds t dt||d D]\}}|||n|D]z}t|tr ||it|trDt |d k(r6t|d tr#t|d tr||d |d qt dtjtjfS)a Validates the 'providers' and 'provider_options' arguments and returns a normalized version. :param providers: Optional sequence of providers in order of decreasing precedence. Values can either be provider names or tuples of (provider name, options dict). :param provider_options: Optional sequence of options dicts corresponding to the providers listed in 'providers'. :param available_provider_names: The available provider names. :return: Tuple of (normalized 'providers' sequence, normalized 'provider_options' sequence). 'providers' can contain either names or names and options. When any options are given in 'providers', 'provider_options' should not be used. The normalized result is a tuple of: 1. Sequence of provider names in the same order as 'providers'. 2. Sequence of corresponding provider options dicts with string keys and values. Unspecified provider options yield empty dicts. c2|vr4tjdj|dj|vrtjd|dy|j Dcic]\}}t |t |}}}||<ycc}}w)NzTSpecified provider '{}' is not in available provider names.Available providers: '{}', zDuplicate provider 'z' encountered, ignoring.)warningswarnformatjoinr^str)nameoptionskeyvaluenormalized_optionsavailable_provider_namesprovider_name_to_optionss rset_provider_optionsz?check_and_normalize_provider_args..set_provider_optionss / / MMfmm$))$<=  + + MM06NO P EL]]_UzsEc#hE 2UU); &Vs+Bz!'providers' should be a sequence.z('provider_options' should be a sequence.zO'providers' and 'provider_options' should be the same length if both are given.c3<K|]}t|tywrC) isinstancerv.0providers r z4check_and_normalize_provider_args..sG:h,GzPOnly string values for 'providers' are supported if 'provider_options' is given.c3<K|]}t|tywrC)rdict)roptions_for_providers rrz4check_and_normalize_provider_args..sg>R:2D9grz('provider_options' values must be dicts.F)strictrzC'providers' values must be either strings or (string, dict) tuples.) collections OrderedDictrabcrr2lenallziprvtuplerlistkeysvalues) providersprovider_optionsr|r~rwrxrr}s ` @r!check_and_normalize_provider_argsrs62v *668 < i!9!9 :<==#*KOO,D,DEGH H y>S!12 2no oGYGGop pgVfggGH H , !fgg h (--/ 0$7O7V7V7X2Y YYrc eZdZdZd$d%dZd&dZd'dZd'dZd'dZd(dZ d)dZ d)d Z d*d Z d+d Z d Zd,d Zd-d.dZd.dZd.dZdZdZd/dZd/d0dZd/dZd/d1dZdZdZd2dZ d3 d4dZd5d6dZd/dZd7dZdZ d d!d"Z!d#Z"y)8Session5 This is the main class used to run a model. c.d|_||_i|_yrC)_sess_enable_fallback_captured_graph_bindings)rEenable_fallbacks rrGzSession.__init__s /NP%rc|jS)zDReturn the session options. See :class:`onnxruntime.SessionOptions`.) _sess_optionsrOs rget_session_optionszSession.get_session_options!!!rc|jS)zEReturn the inputs metadata as a list of :class:`onnxruntime.NodeArg`.) _inputs_metarOs r get_inputszSession.get_inputss   rc|jS)zFReturn the outputs metadata as a list of :class:`onnxruntime.NodeArg`.) _outputs_metarOs r get_outputszSession.get_outputs rrc|jS)z^Return the inputs (including initializers) metadata as a list of :class:`onnxruntime.NodeArg`.)_overridable_initializersrOs rget_overridable_initializersz$Session.get_overridable_initializerss---rc|jS)z>rNc(|j||y)a Register the input list of execution providers. The underlying session is re-created. :param providers: Optional sequence of providers in order of decreasing precedence. Values can either be provider names or tuples of (provider name, options dict). If not provided, then all available providers are used with the default precedence. :param provider_options: Optional sequence of options dicts corresponding to the providers listed in 'providers'. 'providers' can contain either names or names and options. When any options are given in 'providers', 'provider_options' should not be used. The list of providers is ordered by precedence. For example `['CUDAExecutionProvider', 'CPUExecutionProvider']` means execute a node using CUDAExecutionProvider if capable, otherwise execute using CPUExecutionProvider. N)_reset_sessionrErrs r set_providerszSession.set_providers3s( I'78rcd|_y)z; Disable session.run() fallback mechanism. FNrrOs rdisable_fallbackzSession.disable_fallbackIs !&rcd|_y)a4 Enable session.Run() fallback mechanism. If session.Run() fails due to an internal Execution Provider failure, reset the Execution Providers enabled for this session. If GPU is enabled, fall back to CUDAExecutionProvider. otherwise fall back to CPUExecutionProvider. TNrrOs rrzSession.enable_fallbackOs !%rcg}|jD]H}|j|vs|jjdr.|j |jJ|rt d|d|dy)NoptionalzRequired inputs (z) are missing from input feed (z).)rrwtype startswithappendr2)rEfeed_input_namesmissing_input_namesinputs r_validate_inputzSession._validate_inputXs} && 7Ezz!11%**:O:OPZ:[#**5::6 7 #$7#88WXhWiikl  rcb|D]*}t|tst||jd,y)Nz used with)rrfr?r)rErrzs r_validate_ortvalue_ownershipz$Session._validate_ortvalue_ownershipbs0 UEeX. 4UDJJ T Urc|jjsyt|tk(ryt dt dtd)NzVWebGPU graph capture requires fixed device OrtValues and run_with_iobinding. Set the '' run option to z$ to opt a single run out of capture.)ris_webgpu_graph_capture_enabledr5_GRAPH_ANNOTATION_SKIPr2r/)rEr3s r_validate_graph_capture_run_apiz'Session._validate_graph_capture_run_apihsPzz99;   ,0F F  455E%&&J L  rc|j||jt|j|j |j |s#|j Dcgc]}|j}} |jj|||Scc}w#tj$r}|jrtd|d|jtd|jd|j!|j|j#|j |j |jj|||cYd}~Sd}~wwxYw)aD Compute the predictions. WebGPU graph capture requires fixed device OrtValues bound with :meth:`run_with_iobinding`; do not use this convenience API for captured replay. :param output_names: name of the outputs :param input_feed: dictionary ``{ input_name: input_value }`` :param run_options: See :class:`onnxruntime.RunOptions`. :return: list of results, every result is either a numpy array, a sparse tensor, a list or a dictionary. :: sess.run([output_name], {input_name: x}) EP Error:  using Falling back to  and retrying.N)rrrrrrrrwrrunrEPFailrprintr_fallback_providersrr)rE output_names input_feedr3outputerrs rrz Session.runts(" ,,[9 T*//"345 ))**;*;*=>6:6H6HIFFKKILI ::>>, KH HJxx $$ 3'0ABC()A)A(B.QR""4#;#;<%%'11*2C2C2EFzz~~lJ LL  s+)B?B!!E&4B&E!E& E!!E&cD|j||jt|j|j |j |s#|j Dcgc]}|j}}|jj|||||Scc}w)a Compute the predictions asynchronously in a separate cxx thread from ort intra-op threadpool. :param output_names: name of the outputs :param input_feed: dictionary ``{ input_name: input_value }`` :param callback: python function that accept array of results, and a status string on error. The callback will be invoked by a cxx thread from ort intra-op threadpool. :param run_options: See :class:`onnxruntime.RunOptions`. :: class MyData: def __init__(self): # ... def save_results(self, results): # ... def callback(results: np.ndarray, user_data: MyData, err: str) -> None: if err: print (err) else: # save results to user_data sess.run_async([output_name], {input_name: x}, callback) ) rrrrrrrrwr run_async)rErrcallback user_datar3rs rrzSession.run_asyncs2 ,,[9 T*//"345 ))**;*;*=>6:6H6HIFFKKILIzz##L*h S^__Js)Bcd}|j||jt|j|j |j |s#|j Dcgc]}|j}} ||j|||Scc}w#tj$r}|jrtd|d|jtd|jd|j|j|j!|j |j ||j|||cYd}~Sd}~wwxYw)a Compute the predictions. :param output_names: name of the outputs :param input_dict_ort_values: dictionary ``{ input_name: input_ort_value }`` See ``OrtValue`` class how to create `OrtValue` from numpy array or `SparseTensor` :param run_options: See :class:`onnxruntime.RunOptions`. :return: an array of `OrtValue` :: sess.run([output_name], {input_name: x}) c i}|jD]\}}|j||<|j|||}t|tj s t d|Dcgc] }t|}}|Scc}w)NzFrun_with_ort_values() must return a instance of type 'OrtValueVector'.)r^ _get_c_valuerun_with_ort_valuesrrOrtValueVector TypeErrorrf) sessrinput_dict_ort_valuesr3 input_dictnrcresult ort_valuess rinvokez+Session.run_with_ort_values..invokesJ-335 11 ! 0 1  1--j, TFfa&6&67 hii/56!(1+6J6 7s*BrrrrN)rrrrrrrrwrrrrrrrrr)rErrr3rrrs rrzSession.run_with_ort_valuess0   ,,[9 T"7"<"<">?@ ))*?*F*F*HI6:6H6HIFFKKILI $**l4I;W WJxx $$ 3'0ABC()A)A(B.QR""4#;#;<%%'112G2N2N2PQdjj,8M{[[  s+,BBE/BE EEEc6|jjS)z End profiling and return results in a file. The results are stored in a filename if the option :meth:`onnxruntime.SessionOptions.enable_profiling`. )r end_profilingrOs rrzSession.end_profilingszz''))rc.|jjS)a Return the nanoseconds of profiling's start time Comparable to time.monotonic_ns() after Python 3.3 On some platforms, this timer may not be as precise as nanoseconds For instance, on Windows and MacOS, the precision will be ~100ns )rget_profiling_start_time_nsrOs rrz#Session.get_profiling_start_time_nsszz555rct|S)z(Return an onnxruntime.IOBinding object`.) IOBindingrOs r io_bindingzSession.io_bindings rctj|||j}t|tr|j j |||}n|j j|||}t||j S)zCreate an OrtValue using this session's allocator. The value retains this session and may only be updated or bound through it. A matching allocator and a numeric tensor type are required. session) rmake _get_c_devicerr1r(create_ortvalue_from_shape_and_onnx_type#create_ortvalue_from_shape_and_typerf)rEshape element_typer device_id vendor_iddevicer:s rrz+Session.create_ortvalue_from_shape_and_typesm Y BPPR lC (zzJJ5R^`fgHzzEEe\[abH$**55rc|jj||jj|d}|)|d}||jj |yyy)zRelease a captured graph and unpin its IOBinding. ``graph_annotation_id`` matches the capture's ``gpu_graph_id`` and defaults to zero. EPs without captured-graph release support treat this as a no-op. Nr)rrelease_captured_graphrpop_pinned_graph_idsdiscard)rEgraph_annotation_idpinned iobindings rr zSession.release_captured_graph sh ))*=>..223FM  !q I$++334GH% rc|j|jur tdt|}|jj xr |t k7}|ry|j |j}|jj|}|<|d|d}}||urtd|d|d||k7rtd|d|d|jj|j||rO||jvr@tj|f|j|<|jj|yyy) ai Compute the predictions. :param iobinding: the iobinding object that has graph inputs/outputs bind. :param run_options: See :class:`onnxruntime.RunOptions`. WebGPU capture requires static shapes, disabled memory patterns, fixed WebGPU OrtValues (session-owned or shared), and no CPU compute nodes other than shape-only nodes. Each ``gpu_graph_id`` pins one IOBinding and its buffers until :meth:`release_captured_graph`. Update inputs in place and use :meth:`IOBinding.copy_outputs_to_cpu` for readback; ``gpu_graph_id=-1`` disables capture. z8IOBinding must be used with the session that created it.Nrrz WebGPU graph z was captured with a different IOBinding. Replay re-issues the buffers recorded at capture and would silently ignore this binding. Reuse the original IOBinding, or call release_captured_graph(z) first.z was captured with different I/O buffers. Replay writes to the buffers recorded at capture, so the rebound values would never be read or written. Update the original buffers in place, or call release_captured_graph()r7rr2r5rr_validate_capture_bindings_capture_signaturergetrun_with_iobinding _iobindingweakrefrefradd) rErr3r capturing signaturerpinned_iobindingpinned_signatures rrzSession.run_with_iobindingsp   TZZ /WX X2;?JJ>>@rEX\rEr   0 0 2!446I22667JKF!5>VAY[&)"2 #94$'(;'<=23F1FhP $y0$'(;'<=23F1Fh P %%i&:&:KH ,D4Q4QQBI++iBXZcAdD ) )*= >  ' ' + +,? @R9rc:|jj|y)a' Set dynamic options for execution providers. :param options: Dictionary of key-value pairs where both keys and values are strings. These options will be passed to the execution providers to modify their runtime behavior. N)rset_ep_dynamic_options)rErxs rr"zSession.set_ep_dynamic_optionsHs ))'2rc6|jjSrC)rget_tuning_resultsrOs rr$zSession.get_tuning_resultsRszz,,..rF)error_on_invalidc:|jj||SrC)rset_tuning_results)rEresultsr%s rr'zSession.set_tuning_resultsUszz,,W6FGGrcf|j||jj||||||y)a Compute the predictions similar to other run_*() methods but with minimal C++/Python conversion overhead. :param run_options: See :class:`onnxruntime.RunOptions`. :param feed_names: list of input names. :param feeds: list of input OrtValue. :param fetch_names: list of output names. :param fetches: list of output OrtValue. :param fetch_devices: list of output devices. N)rrrun_with_ortvaluevector)rEr3 feed_namesfeeds fetch_namesfetches fetch_devicess rr*zSession.run_with_ortvaluevectorXs0 ,,[9 **; E;X_anor)T)rbool)riC.SessionOptions)rizSequence[C.NodeArg])rizC.ModelMetadata)rizSequence[C.OrtMemoryInfo])rizSequence[C.OrtEpDevice])ri Sequence[str])riz!Sequence[C.OrtEpAssignedSubgraph]NNrhrC)riz1Sequence[np.ndarray | SparseTensor | list | dict])riSequence[OrtValue])rir)rr+) r Sequence[int]rrvr r1r int | OrtDeviceVendorIdrirf)r)rr1rirj)rxzdict[str, str])#rrrrrGrrrrrrrrrrrrrrrrrrrrrrrrr rr"r$r'r*r'rrrrsQ"!". $%%&?9,& %U  !F`@*X*6-/ 66 6  6 + 6 6( I,A\3/?DHprrc\eZdZdZ d dfd Zd dZd dZd dZdZxZ S) InferenceSessionrc t |t|jdddk(t |t t jfr"t j||_ d|_ n7t |trd|_ ||_ ntdt|d||_||_d|vrt|ddk(|_n't j"jdd k(|_|jd } |j%|||y#t&t(f$r}|j*r t-d t-d |d |t-d|j.dt-d|j%|j.d|j1Yd}~y#t2$r}||d}~wwxYw|d}~wwxYw)a :param path_or_bytes: Filename or serialized ONNX or ORT format model in a byte string. :param sess_options: Session options. :param providers: Optional sequence of providers in order of decreasing precedence. Values can either be provider names or tuples of (provider name, options dict). If not provided, then all available providers are used with the default precedence. :param provider_options: Optional sequence of options dicts corresponding to the providers listed in 'providers'. The model type will be inferred unless explicitly set in the SessionOptions. To explicitly set: :: so = onnxruntime.SessionOptions() # so.add_session_config_entry('session.load_model_format', 'ONNX') or so.add_session_config_entry('session.load_model_format', 'ORT') A file extension of '.ort' will be inferred as an ORT format model. All other filenames are assumed to be ONNX format models. 'providers' can contain either names or names and options. When any options are given in 'providers', 'provider_options' should not be used. The list of providers is ordered by precedence. For example `['CUDAExecutionProvider', 'CPUExecutionProvider']` means execute a node using `CUDAExecutionProvider` if capable, otherwise execute using `CPUExecutionProvider`. rr)rNUnable to load from type ''read_config_from_modelORT_LOAD_CONFIG_FROM_MODEL1disabled_optimizersz(*************** EP Error ***************z EP Error z when using rrz(****************************************)superrGr1rrrvosPathLikefspath _model_path _model_bytesbytesrrr_sess_options_initial_read_config_from_modelenviron_create_inference_sessionr2r0rrrrr) rE path_or_bytes sess_optionsrrkwargsr?efallback_error __class__s rrGzInferenceSession.__init__osL VZZ8I1-M)NRS)ST mc2;;%7 8!yy7D  $D   u -#D  -D 8m9L8MQOP P)%1" #v -+.v6N/O+PTU+UD (+-::>>:V+W[^+^D (%jj)>?   * *96FH[ \L) $$ 0DEIaS YK@A,T-E-E,FnUVDE2243K3KTR))+ 0(a/0G s7 DG, G9A,F** F;3F66F;;GGchtj}|r3td|D}td|D}|r |r tdd|vr9|r.td|Drtd|Dr ddg|_nNdg|_nEd |vr9|r.td |Drtd |Dr ddg|_ndg|_ndg|_t |||\}}|j 3|s|r/|j jrtjd |j r |j ntj}|j|||||jr-tj||jd |j}n,tj||jd|j}| t!}nt#|t s t!|}|j%|||||_|j&j(|_|j&j*|_|j&j.|_|j&j2|_|j&j6|_|j&j:|_|j&j>|_ |j&jB|_"|j&jG|_$|j&jK|_&|j&jN|_(y)Nc3^K|]%}|dk(xst|txr|ddk('ywTensorrtExecutionProviderrNrrrs rrz=InferenceSession._create_inference_session..sE77`x/^HQKC^4^`+-c3^K|]%}|dk(xst|txr|ddk('ywNvTensorRTRTXExecutionProviderrNrUrs rrz=InferenceSession._create_inference_session..sE#<<ex/cHQKCc4ce#rVzhCannot enable both 'TensorrtExecutionProvider' and 'NvTensorRTRTXExecutionProvider' in the same session.rYc3^K|]%}|dk(xst|txr|ddk('ywCUDAExecutionProviderrNrUrs rrz=InferenceSession._create_inference_session..E! 77`"8U3^ G^8^`rVc3^K|]%}|dk(xst|txr|ddk('ywrXrUrs rrz=InferenceSession._create_inference_session..sE! @@i"8U3g Gg8girVr\CPUExecutionProviderrTc3^K|]%}|dk(xst|txr|ddk('ywr[rUrs rrz=InferenceSession._create_inference_session..r]rVc3^K|]%}|dk(xst|txr|ddk('ywrSrUrs rrz=InferenceSession._create_inference_session..sE! ;;d"8U3b Gb8bdrVzSpecified 'providers'/'provider_options' when creating InferenceSession but SessionOptions has already been configured with providers. InferenceSession will only use the providers passed to InferenceSession().TF))rget_available_providersanyr2rrr has_providersrrrsget_default_session_options_register_ep_custom_opsrDr8rHrEsetrinitialize_sessionrsession_options inputs_metar outputs_metaroverridable_initializersrinput_meminfosroutput_meminfosrinput_epdevicesr model_metarrrrrr_profiling_start_time_ns) rErrr?available_providers has_tensorrthas_tensorrt_rtxrirs rrJz*InferenceSession._create_inference_sessions779 !*L ##!*#    0 + ,/B B%. %. -DE[+\(,B+C( (,? ?%. %. -DE[+\(,B+C((>'?D $'H ')<' # #    )y.get_initializer_location_func_wrapperjsZ 8U h'89=8   7m#;2273C3CWEXEXZaZkZklGr)rrvr C.OrtValuer#C.OrtExternalInitializerInfo | Nonerirr')rrs` r*make_get_initializer_location_func_wrapperr]s4  % ;  -  10rceZdZdZdddej j ejjdf d dZ d d dZ ddZ ddZ dd Z dd Zy) ModelCompilerag This class is used to compile an ONNX model. A compiled ONNX model has EPContext nodes that each encapsulates a subgraph compiled/optimized for a specific execution provider. Refer to the EPContext design document for more information about EPContext models: https://onnxruntime.ai/docs/execution-providers/EP-Context-Design.html :: sess_options = onnxruntime.SessionOptions() sess_options.add_provider("SomeExecutionProvider", {"option1": "value1"}) # Alternatively, allow ONNX Runtime to select the provider automatically given a policy: # sess_options.set_provider_selection_policy(onnxrt.OrtExecutionProviderDevicePolicy.PREFER_NPU) model_compiler = onnxruntime.ModelCompiler(sess_options, "input_model.onnx") model_compiler.compile_to_file("output_model.onnx") FNic d} d} t|ttjfr#|s t dtj |} nDt|t rt|dk(r t d|} ntdt|d|rPt|ttjfst|} td| dtj |}nd}||r t d t||_ nd|_ | r-tj|| d ||||||j |_ytj|| d ||||||j |_y) a Creates a ModelCompiler instance. :param sess_options: Session options containing the providers for which the model will be compiled. Refer to SessionOptions.add_provider() and SessionOptions.set_provider_selection_policy(). :param input_model_path_or_bytes: The path to the input model file or bytes representing a serialized ONNX model. :param embed_compiled_data_into_model: Defaults to False. Set to True to embed compiled binary data into EPContext nodes in the compiled model. :param external_initializers_file_path: Defaults to None. Set to a path for a file that will store the initializers for non-compiled nodes. :param external_initializers_size_threshold: Defaults to 1024. Ignored if `external_initializers_file_path` is None or empty. Initializers larger than this threshold are stored in the external initializers file. :param flags: Additional boolean options to enable. Set this parameter to a bitwise OR of flags in onnxruntime.OrtCompileApiFlags. :param graph_optimization_level: The graph optimization level. Defaults to onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL. :param get_initializer_location_func: Optional function called for every initializer to allow user to specify whether an initializer should be stored within the model or externally. Example: ``` def get_initializer_location( initializer_name: str, initializer_value: onnxrt.OrtValue, external_info: onnxrt.OrtExternalInitializerInfo | None, ) -> onnxrt.OrtExternalInitializerInfo | None: byte_size = initializer_value.tensor_size_in_bytes() if byte_size < 64: return None # Store small initializer within compiled model. # Else, write initializer to new external file. value_np = initializer_value.numpy() file_offset = ext_init_file.tell() ext_init_file.write(value_np.tobytes()) return onnxrt.OrtExternalInitializerInfo(initializer_file_path, file_offset, byte_size) ``` NzInput model path is emptyrz Input model bytes array is emptyr:r;zW9X8YYZ[\ \ *=R[[?QR ?@"^_g^hhi jkk.0ii8W.X +.0 + ( 4. ::d-:D 6:>D 6 #$?? ./4(:: $D $%??!./4(:: $D rc|rMt|ttjfst dt |dtj |}|jj|y)a Compiles to an output file. If an output file path is not provided, the output file path is generated based on the input model path by replacing '.onnx' with '_ctx.onnx'. Ex: The generated output file is 'model_ctx.onnx' for an input model with path 'model.onnx'. Raises an 'InvalidArgument' exception if the compilation options are invalid. :param output_model_path: Defaults to None. The path for the output/compiled model. z/Output model's filepath is of unexpected type 'r;N) rrvrArBrrrCrcompile_to_file)rEoutput_model_paths rrzModelCompiler.compile_to_files^ /#r{{1CD"QRVWhRiQjjk lmm " *; <  ,,->?rc:|jj|y)a Registers a callback that receives external EPContext data during compilation. The ``OrtEpContextData`` view is valid only during the callback. Read large payloads in chunks with ``data.read(offset, length)``. ORT may invoke the callback concurrently, so synchronize shared state. :param write_func: Callback receiving the logical data name and a bounded data view. N)rset_ep_context_data_write_func)rE write_funcs rrz,ModelCompiler.set_ep_context_data_write_func s ;;JGrc8|jjy)zFClears a previously registered external EPContext data write callback.N)r clear_ep_context_data_write_funcrOs rrz.ModelCompiler.clear_ep_context_data_write_funcs ==?rc6|jjS)z Compiles to bytes representing the serialized compiled ONNX model. Raises an 'InvalidArgument' exception if the compilation options are invalid. :return: A bytes object representing the compiled ONNX model. )rcompile_to_bytesrOs rrzModelCompiler.compile_to_bytess##4466rc:|jj|y)a$ Compiles the input model and writes the serialized ONNX bytes to a stream using the provided write function. Raises an 'InvalidArgument' exception if the compilation options are invalid. :param write_function: A callable that accepts a bytes buffer to write. N)rcompile_to_stream)rEwrite_functions rrzModelCompiler.compile_to_stream"s ..~>r)rLr1rzstr | os.PathLike | bytesrr0rzstr | os.PathLike | Nonerr1rr1rzC.GraphOptimizationLevelrz!GetInitializerLocationFunc | NonerC)rz str | None)rz)Callable[[str, C.OrtEpContextData], None]rirjrh)rirF)rzCallable[[bytes], None])rrrrrOrtCompileApiFlagsrGraphOptimizationLevelORT_DISABLE_ALLrGrrrrrr'rrrr{s,05DH48))..=>=U=U=e=eKOh&h$=h)- h *B h /2 hh#;h(IhT@" H@7?rrceZdZdZddZddZdZddZdZdZ dZ d Z dd Z d Z d ZdZdZdZdZdZy )rzW This class provides API to bind input/output to a specified device, e.g. GPU. c"|j|_d|jv|_|jj |_t j|j|_i|_ i|_ i|_ t|_ y)NWebGpuExecutionProvider)rr7r_is_webgpu_sessionr _is_webgpu_graph_capture_enabledrSessionIOBindingr_numpy_obj_references _bound_inputs_bound_outputsrgr)rErs rrGzIOBinding.__init__0sp  ";w?T?T?V"V04 0]0]0_-,,W]];%'"9;:<+.5rc|jr=djdt|jD}td|d|dy)Nrqc32K|]}t|ywrC)rv)rgraph_ids rrz.IOBinding._reject_if_pinned..>s\s8}\szCannot z! while captured WebGPU graph(s) [z] still reference this IOBinding. Replay re-issues the buffers recorded at capture, so a change here would not affect replay and releasing the buffers could invalidate it. Call release_captured_graph(id) first.)rrurvr2)rEr>rs r_reject_if_pinnedzIOBinding._reject_if_pinned<sU  ! !YY\VDDZDZ=[\\F&!B6(K44  "rctt|jjdtt|jjdfS)zSReturn bound OrtValues whose identity and lifetime must remain fixed during replay.c |dSNrr'items rz.IOBinding._capture_signature..Is d1gr)ryc |dSrr'rs rrz.IOBinding._capture_signature..Js tAwr)rrvrr^rrOs rrzIOBinding._capture_signatureFsL &++1139MN O &,,224:NO P  rc d|jfd|jffD]K\}}|jD]3\}}| |jrt d|d|dt dt d My) zNEvery binding participating in capture must be a fixed WebGPU device OrtValue.rrNz=WebGPU graph capture requires fixed WebGPU device OrtValues; z 'zw' is not one. Bind values created by this session or backed by an environment-registered shared allocator, or set the 'rz to run without capture.)rrr^r8r2r/r)rEkindboundrwrzs rrz$IOBinding._validate_capture_bindingsMs$d&8&89HdFYFY;Z[ KD%${{}  e=(?(?$WX\W]^ 6"899IJ`Iab22  rc|jd||j|<|jj||d|j|<y)z bind an input to array on CPU :param name: input name :param arr_on_cpu: input values as a python array on CPU rebind inputsN)rrr bind_inputr)rErw arr_on_cpus rbind_cpu_inputzIOBinding.bind_cpu_inputZsH / ,6""4( ""44#'4 rc|jd|jj|tj ||j |||d|j |<y)a :param name: input name :param device_type: e.g. cpu, cuda, cann :param device_id: device id, e.g. 0 :param element_type: input element type. It can be either numpy type (like numpy.float32) or an integer for onnx type (like onnx.TensorProto.BFLOAT16) :param shape: input shape :param buffer_ptr: memory pointer to input data rN)rrrrrrrrErwrr rr buffer_ptrs rrzIOBinding.bind_inputisY / ""  NN; 2 @ @ B     $(4 rct||jd|jd|jj ||j ||j |<y)z\ :param name: input name :param ortvalue: OrtValue instance to bind bound torN)r?r7rrbind_ortvalue_inputr_rrErwr:s rrzIOBinding.bind_ortvalue_input|sL 14==*U / ++D(2D2DE#+4 rc8|jjyrC)rsynchronize_inputsrOs rrzIOBinding.synchronize_inputss **,rNcf|jd|?|jj|tj ||j nP|| t d|jj|tj ||j |||d|j|<y)a :param name: output name :param device_type: e.g. cpu, cuda, cann, cpu by default :param device_id: device id, e.g. 0 :param element_type: output element type. It can be either numpy type (like numpy.float32) or an integer for onnx type (like onnx.TensorProto.BFLOAT16) :param shape: output shape :param buffer_ptr: memory pointer to output data rebind outputsNzQ`element_type` and `shape` are to be provided if pre-allocated memory is provided)rr bind_outputrrrr2rrs rrzIOBinding.bind_outputs$ /0   OO ' '{I6DDF  #u} !tuu OO ' '{I6DDF  %)D!rct||jd|jd|jj ||j ||j |<y)z] :param name: output name :param ortvalue: OrtValue instance to bind rrN)r?r7rrbind_ortvalue_outputr_rrs rrzIOBinding.bind_ortvalue_outputsM 14==*U /0 ,,T83E3EF$,D!rc8|jjyrC)rsynchronize_outputsrOs rrzIOBinding.synchronize_outputss ++-rc4|jj}t|tjs t dg}t t|D]?}||}|jt||jr |jndA|S)z Returns the output OrtValues from the Run() that preceded the call. The data buffer of the obtained OrtValues may not reside on CPU memory z?get_outputs() must return an instance of type 'OrtValueVector'.Nr) rrrrrrr{rrrfr8r7)rEoutputsrindexr:s rrzIOBinding.get_outputss //--/'1#3#34]^ ^3w<( Eu~H MM-5-G-G-IDMMt   rc6|jjS)a`Return the raw OrtValueVector of outputs from the Run() that preceded the call. The vector is a reference into this IOBinding (pybind ``reference_internal``), so it keeps the IOBinding alive, which in turn keeps the session alive. Device-resident outputs are therefore safe to hold past the session going out of scope. )rrrOs rget_outputs_as_ortvaluevectorz'IOBinding.get_outputs_as_ortvaluevectors**,,rc6|jjS)zCopy output contents to CPU.)rcopy_outputs_to_cpurOs rrzIOBinding.copy_outputs_to_cpus2244rc|jd|jj|jj y)Nzclear input bindings)rrclear_binding_inputsrclearrOs rrzIOBinding.clear_binding_inputss3 56 ,,.   "rc|jd|jj|jj y)Nzclear output bindings)rrclear_binding_outputsrrrOs rrzIOBinding.clear_binding_outputss3 67 --/ !!#r)rr)r>rvrirjrh)r rNNN)rrrrrGrrrrrrrrrrrrrrrr'rrrr+si 1   ((&,- ))V-.&-5# $rrcPeZdZdZ d ddZeddZddZe d d!dZ ed"dZ e d# d$dZ ed%d Z d&d Z d'd Zd(d Zd)d Zd(dZd'dZd'dZddZddZddZddZd*dZdd*dZdddZd+dZed,dZd-dZy).rfa A data structure that supports all ONNX data formats (tensors and non-tensors) that allows users to place the data backing these on a device, for example, on a CUDA supported device. This class provides APIs to construct and deal with OrtValues. Ncxt|tjr||_||_||_yt d)Nz^`Provided ortvalue` needs to be of type `onnxruntime.capi.onnxruntime_pybind11_state.OrtValue`)rrrfr_ _numpy_objr7r2)rEr: numpy_objrs rrGzOrtValue.__init__s; h +%DN(DO#DMp rc6|jjS)a?Whether this value is backed by a WebGPU buffer. Resolved from the native OrtValue on every access and deliberately read-only: graph-capture validation and the copy-path checks below refuse host memory, so a settable attribute would let a CPU tensor pass itself off as a device tensor. )r_r8rOs rr8zOrtValue._is_webgpu_buffers~~//11rc|jSrC)r_rOs rrzOrtValue._get_c_values ~~rc |tjj|tj |||j |j dk(r|SdS)a Factory method to construct an OrtValue (which holds a Tensor) from a given Numpy object A copy of the data in the Numpy object is held by the OrtValue only if the device is NOT cpu :param numpy_obj: The Numpy object to construct the OrtValue from :param device_type: e.g. cpu, cuda, cann, cpu by default :param device_id: device id, e.g. 0 :param vendor_id: The device's PCI vendor id as an int or OrtDeviceVendorId. If provided, the device_type should be "gpu" or "npu". r N)rrfortvalue_from_numpyrrrlower)clsrrr r s rrzOrtValue.ortvalue_from_numpysd  JJ * *{IyAOOQ %**,5I   <@   rcP|tjj|||S)a  This method creates an instance of OrtValue on top of the numpy array. No data copy is made and the lifespan of the resulting OrtValue should never exceed the lifespan of bytes object. The API attempts to reinterpret the data type which is expected to be the same size. This is useful when we want to use an ONNX data type that is not supported by numpy. :param data: numpy.ndarray. :param onnx_element_type: a valid onnx TensorProto::DataType enum value )rrf"ortvalue_from_numpy_with_onnx_type)rdataonnx_element_types rrz+OrtValue.ortvalue_from_numpy_with_onnx_type+s$1::@@GXY[_``rc||dk7s |dk7s|dk7rtjddt|tr'|tj j |||S|tj j|||Stj|||j}t|tr'|tj j|||S|tj j|||S)a Factory method to construct an OrtValue (which holds a Tensor) from given shape and element_type :param shape: List of integers indicating the shape of the OrtValue :param element_type: The data type of the elements. It can be either numpy type (like numpy.float32) or an integer for onnx type (like onnx.TensorProto.BFLOAT16). :param device_type: e.g. cpu, cuda, cann, cpu by default :param device_id: device id, e.g. 0 :param vendor_id: The device's PCI vendor id as an int or OrtDeviceVendorId. If provided, the device type should be "gpu" or "npu". :param memory_info: An OrtMemoryInfo from an OrtEpDevice (e.g. via ep_device.memory_info(OrtDeviceMemoryType.HOST_ACCESSIBLE)). When provided, the allocator matching this memory info is used directly, which allows allocating HOST_ACCESSIBLE memory for zero-copy numpy interop. The device_type, device_id, and vendor_id parameters are ignored when memory_info is provided. r rr+zOdevice_type, device_id, and vendor_id are ignored when memory_info is provided.r) stacklevel) rrrsrr1rrf1ortvalue_from_shape_and_onnx_type_for_memory_info,ortvalue_from_shape_and_type_for_memory_inforrr!ortvalue_from_shape_and_onnx_typeortvalue_from_shape_and_type)rrrrr r  memory_infor s rrz%OrtValue.ortvalue_from_shape_and_type9s *  "e#yA~b e ,,JJPP$# GG   Y BPPR lC ( <<   JJ 3 3   rch|tjj|jS)z The function will construct an OrtValue instance from a valid SparseTensor The new instance of OrtValue will assume the ownership of sparse_tensor )rrfort_value_from_sparse_tensor _get_c_tensor)r sparse_tensors rrz%OrtValue.ort_value_from_sparse_tensorys' 1::::=;V;V;XYZZrcHt|jjS)zR The function will return SparseTensor contained in this OrtValue ) SparseTensorr_as_sparse_tensorrOs rrzOrtValue.as_sparse_tensorsDNN;;=>>rc6|jjS)z Returns the address of the first element in the OrtValue's data buffer WebGPU buffers are opaque handles and do not expose a data pointer. )r_data_ptrrOs rrzOrtValue.data_ptrs ~~&&((rcR|jjjS)zv Returns the name of the device where the OrtValue's data buffer resides e.g. cpu, cuda, cann, webgpu )r_ device_namerrOs rrzOrtValue.device_names ~~))+1133rc6|jjS)z? Returns the shape of the data in the OrtValue )r_rrOs rrzOrtValue.shapes~~##%%rc6|jjS)zY Returns the data type of the data in the OrtValue. E.g. 'tensor(int64)' )r_ data_typerOs rr zOrtValue.data_type~~''))rc6|jjS)zi Returns the proto type of the data in the OrtValue if the OrtValue is a tensor. )r_rrOs rrzOrtValue.element_types ~~**,,rc6|jjS)zl Returns the size of the data in the OrtValue in bytes if the OrtValue is a tensor. )r_tensor_size_in_bytesrOs rr zOrtValue.tensor_size_in_bytess ~~2244rc6|jjS)zz Returns True if the OrtValue corresponding to an optional type contains data, else returns False )r_ has_valuerOs rrzOrtValue.has_values ~~''))rc6|jjS)zT Returns True if the OrtValue contains a Tensor, else returns False )r_ is_tensorrOs rrzOrtValue.is_tensorr rc6|jjS)zZ Returns True if the OrtValue contains a SparseTensor, else returns False )r_is_sparse_tensorrOs rrzOrtValue.is_sparse_tensors~~..00rc6|jjS)z] Returns True if the OrtValue contains a Tensor Sequence, else returns False )r_is_tensor_sequencerOs rrzOrtValue.is_tensor_sequences~~0022rc6|jjS)aW Returns a Numpy object from the OrtValue. Valid only for OrtValues holding Tensors. Throws for OrtValues holding non-Tensors. Use accessors to gain a reference to non-Tensor objects such as SparseTensor WebGPU device values require explicit readback with :meth:`IOBinding.copy_outputs_to_cpu`. )r_numpyrOs rrzOrtValue.numpys~~##%%rc|j}|tj|||}|S|tj||}|S)a` Supports ``numpy.asarray(ortvalue)`` and ``numpy.array(ortvalue)`` via the `numpy __array__ protocol `_. Valid only for OrtValues holding Tensors on CPU. :param dtype: Optional numpy dtype to cast the result to. :param copy: Optional bool (numpy >= 2.0). If ``False``, a copy will only be made if necessary. If ``True``, a copy is always forced. If ``None`` (default), a copy will be made only if needed. :return: A numpy array with the same data as the OrtValue. )dtypecopy)r)rnparrayasarray)rErrarrs r __array__zOrtValue.__array__sOjjl  ((3e$7C   **S.C rstreamc:|jj|S)ar Returns a DLPack capsule representing the tensor (part of the `DLPack protocol `_). This enables interoperability with other frameworks via ``from_dlpack(ortvalue)`` (e.g. ``torch.from_dlpack``, ``jax.dlpack.from_dlpack``, ``numpy.from_dlpack``). The OrtValue must hold a contiguous tensor. No data is copied; the consumer shares memory with this OrtValue, which must remain alive while the capsule is in use. WebGPU OrtValues cannot be exported through DLPack because their data is stored in opaque WebGPU buffers rather than CUDA-addressable memory. :param stream: Optional stream on which the tensor data is accessible. Currently unused; included for protocol compliance. :return: A PyCapsule holding a DLManagedTensor. r )r_ __dlpack__)rEr!s rr#zOrtValue.__dlpack__s&~~(((77rc6|jjS)am Returns ``(device_type, device_id)`` indicating where the tensor data resides (part of the `DLPack protocol `_). WebGPU OrtValues do not expose a DLPack device. :return: Tuple of ``(device_type, device_id)`` as ints following DLPack ``DLDeviceType`` enum values. )r___dlpack_device__rOs rr%zOrtValue.__dlpack_device__s~~//11rc<d}t|tr|jdk(}n2t|dr&|j}t |dt |}|dv}t|dr|j}n|}|tjj||S)a Construct an OrtValue from an object that implements the DLPack protocol. Accepts either: * An object with ``__dlpack__`` / ``__dlpack_device__`` methods (e.g. a PyTorch tensor, JAX array, or numpy array). * A raw DLPack PyCapsule (legacy path). Boolean tensors are automatically detected when the source object exposes a ``dtype`` attribute (numpy, PyTorch, etc.) or is an ``OrtValue``. For raw DLPack capsules where the original dtype cannot be inspected, bool tensors encoded as uint8 by older DLPack versions are not distinguishable from true uint8 tensors and will be imported as uint8. No data is copied; the new OrtValue shares memory with the source. :param data: A tensor object supporting the DLPack protocol, or a raw DLPack PyCapsule. :return: An OrtValue wrapping the tensor data. :raises TypeError: If the object does not provide a valid DLPack capsule. Fz tensor(bool)rrw)r0bool_z torch.boolr#) rrfr hasattrrgetattrrvr#r from_dlpack)rris_bool dtype_obj dtype_namecapsules rr*zOrtValue.from_dlpacks8 dH %nn&.8G T7 # I!FC NCJ $CCG 4 &oo'GG1::))'7;<Session-scoped OrtValues must originate from the same session.Nz,data must be a numpy.ndarray or an OrtValue.)rrfr8r2r<r7r_update_inplacerndarrayrr c_contiguousascontiguousarray)rErs rr0zOrtValue.update_inplace<s dH %%%)?)?? F34G !abb NN ) )$.. 9 $ +JK Kzz&&''-D %%d+rr3)r:rrznp.ndarray | NonerzC.InferenceSession | Nonerir0)rir)r rr+)r r6r np.ndarrayrirf)rr1rr5rirf)r rr+N) rr5rrvr r1r r6rzC.OrtMemoryInfo | Nonerirf)rrrirf)rirrlrirv)rir5rir5)riztuple[int, int])rirfrh)rrrrrGpropertyr8r classmethodrrrrrrrrr rr rrrrrrr#r%r*r0r'rrrfrfsk(,-1 %+ &22ln Ri "   . a a !-/.2= =  =  = + = ,=  = = ~[[? )4 & * -5** 1 3 &4$(8* 2+=+=Z,rrfc"|jS)zEReturn a session's WebGPU context id, or -1 when it has no WebGPU EP.)r9rs r_session_webgpu_context_idr;`s  $ $ &&rc||D]7}|jr |j||jdtfvs7yy)aWhether any value provably belongs to a WebGPU context other than the default one. The environment data transfer resolves the default WebGPU context, so copying a buffer that belongs to a caller-supplied context would silently run on the wrong device. A value with no owning session cannot be attributed to a context and is treated as default-context, as is a session with no WebGPU EP. TODO: this can only attribute a value that carries session provenance, because every WebGPU allocation reports OrtDevice(GPU, VendorIds::NONE, 0) regardless of its context. Supporting a custom external WebGPU context requires the shared allocator and its data transfer to retain and use that external device instead of resolving WebGpuContextFactory::DefaultContext(), which would also let the transfer itself reject the copy instead of relying on this Python-side guard. r+TF)r8r7_DEFAULT_WEBGPU_CONTEXT_ID)rcontext_id_getterrzs r_has_foreign_webgpu_contextr?esG"&&%..*@  U^^ ,R9S4T T  rctg||r td|Dcgc]}|j}}|Dcgc]}|j}}tj|||ycc}wcc}w)a1 Copy tensor data from source OrtValue sequence to destination OrtValue sequence. WebGPU values belonging to the default context are supported in either direction. A value from a caller-supplied WebGPU context is not, because the shared data transfer is bound to the default context. zcopy_tensors cannot copy a WebGPU OrtValue that belongs to a custom WebGPU context because the shared data transfer is bound to the default context; use IOBinding readback instead.N)r?r2rr copy_tensors)srcdstr!s c_sourcesdc_dstss rrArA~st#>77 V[ZdZrZrZtuvvrc p|tjj|||||jS)a` Factory method to construct a SparseTensor in CSR format from given arguments :param dense_shape: 1-D numpy array(int64) or a python list that contains a dense_shape of the sparse tensor (rows, cols) must be on cpu memory :param values: a contiguous, homogeneous 1-D numpy array that contains non-zero elements of the tensor of a type. :param inner_indices: contiguous 1-D numpy array(int64) that contains CSR inner indices for the tensor. Its length must be equal to that of the values. :param outer_indices: contiguous 1-D numpy array(int64) that contains CSR outer indices for the tensor. Its length must be equal to the number of rows + 1. :param ort_device: - describes the backing memory owned by the supplied nummpy arrays. Only CPU memory is suppored for non-numeric data types. For primitive types, the method will map values and indices arrays into native memory and will use them as backing storage. It will increment the reference count and it will decrement then count when it is GCed. The buffers may reside in any storage either CPU or GPU. For strings and objects, it will create a copy of the arrays in CPU memory as ORT does not support those on other devices and their memory can not be mapped. )rrsparse_csr_from_numpyr)rr`r inner_indices outer_indicesrbs rrdz"SparseTensor.sparse_csr_from_numpys;: NN 0 0((*    rc6|jjS)z The method returns a numpy array that is backed by the native memory if the data type is numeric. Otherwise, the returned numpy array that contains copies of the strings. )r\rrOs rrzSparseTensor.values7s ||""$$rc6|jjS)a} The method will return coo representation of the sparse tensor which will enable querying COO indices. If the instance did not contain COO format, it would throw. You can query coo indices as: :: coo_indices = sparse_tensor.as_coo_view().indices() which will return a numpy array that is backed by the native memory. )r\ get_coo_datarOs r as_coo_viewzSparseTensor.as_coo_view?s||((**rc6|jjS)a The method will return CSR(C) representation of the sparse tensor which will enable querying CRS(C) indices. If the instance dit not contain CSR(C) format, it would throw. You can query indices as: :: inner_ndices = sparse_tensor.as_csrc_view().inner() outer_ndices = sparse_tensor.as_csrc_view().outer() returning numpy arrays backed by the native memory. )r\ get_csrc_datarOs r as_csrc_viewzSparseTensor.as_csrc_viewMs||))++rc6|jjS)a The method will return coo representation of the sparse tensor which will enable querying BlockSparse indices. If the instance did not contain BlockSparse format, it would throw. You can query coo indices as: :: block_sparse_indices = sparse_tensor.as_blocksparse_view().indices() which will return a numpy array that is backed by the native memory )r\get_blocksparse_datarOs ras_blocksparse_viewz SparseTensor.as_blocksparse_view\s||0022rcft|jj|jS)a Returns a copy of this instance on the specified cuda device :param ort_device: with name 'cuda' and valid gpu device id The method will throw if: - this instance contains strings - this instance is already on GPU. Cross GPU copy is not supported - CUDA is not present in this build - if the specified device is not valid )rr\to_cudar)rErbs rrrzSparseTensor.to_cudajs'DLL001I1I1KLMMrc.|jjS)z7 Returns a OrtSparseFormat enumeration )r\rtrOs rrtzSparseTensor.formatys||"""rc6|jjS)zZ Returns a numpy array(int64) containing a dense shape of a sparse tensor )r\r`rOs rr`zSparseTensor.dense_shapes||''))rc6|jjS)zH Returns a string data type of the data in the OrtValue )r\r rOs rr zSparseTensor.data_types||%%''rcR|jjjS)zj Returns the name of the device where the SparseTensor data buffers reside e.g. cpu, cuda )r\rrrOs rrzSparseTensor.device_names ||'')//11rN)rC.SparseTensor)rirw) r`npt.NDArray[np.int64]rr5rarxrbrrir) r`rxrr5rerxrfrxrbrrirr7)rirxr6)rrrrrGrr9r_rdrrjrmrprrrtr`r rr'rrrrs w*ww+ w  w  ww:$ *$ $ - $ - $  $  $ $ L% + , 3 N# * ( 2rr)rrvrir1)rrvrizOrtDeviceVendorId | Nonerlr4rh)rrrrr|r2)rGetInitializerLocationFuncri!GetInitializerLocationWrapperFunc)rzC.InferenceSessionrir1)rr4rir0rC)rBr4rCr4rirj)/ __future__rrcollections.abcrAtypingrrrrrenumrrrronnxruntime.capirr TYPE_CHECKING numpy.typingnptrrr)r/rr5r<r?rArrr8rrrrfr=r;r?rArrrvrryrzr'rrrs # ./ C&    . v vc1M1MhPZ@PZ5PZ,PZfqpqph pcwpcf1#=1&1