VhR dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZddlZddlmZmZmZmZmZmZmZmZmZmZddlmZddlmZej@e!Z"e dZ#e#jIZ%Gd d e&Z' d)d ed e(d e(dee)e(e ffdZ*dZ+dddddddddddd d eedeejXjZdee.e dee)e(e(ffdZ/dddddZ0dddddee)e(e(ffdZ1dZ2dZ3dZ4dZ5 d*d!eejXjZfd"Z6d#Z7ddd$dddd%d d&dee)e(e(fd'ee8e(ffd(Z9y)+a Utilities for debugging and reproducing issues in Ahead of Time with Inductor (AOTI) compilation. This file provides tools and utilities for: - Generating minimal reproducible test cases (minification) - Handling exported programs and graph modules - Creating debug repros for AOTI compilation issues - Supporting both accuracy testing and error reproduction - Managing configuration and environment for repro cases The main components include: - Minification tools to reduce test cases while preserving errors - Repro generation utilities for exported programs - Error handling specific to AOTI compilation - Command-line interface for running and managing repros N) import_module)AnyOptionalUnion) _cuda_system_info_commentBuckTargetWriter extra_importsgenerate_config_stringgenerate_env_vars_stringhelper_for_dump_minify InputReader minifier_dirNNModuleToStringNopInputReader)ExportedProgram)tqdmztorch._inductor.configceZdZfdZxZS)AOTIMinifierErrorcVd}|dt|}t| |||_y)NzRThis error is caused by a bug in the AOTI minifier, please report a bug to PyTorchz: )strsuper__init__original_exception)selfradditional_message full_message __class__s H/home/dcms/DCMS/lib/python3.12/site-packages/torch/_dynamo/repro/aoti.pyrzAOTIMinifierError.__init__9s6q,-R4F0G/HI  &"4)__name__ __module__ __qualname__r __classcell__)rs@rrr8s 55rrexported_program compiler_namecommandoptionsc |dvsJtjjtd}tjj |stj |d|dk(r>t j}t||||d|t|jStj}tjj|d} t|d5}t|||||d d d d d tjd |trt!|j#y y #1swY@xYw#t$$rtjd |Yy wxYw)z If command is "minify": Dump exported_program to `debug_dir/minifier/minifier_launcher.py`, with minify command. If command is "run": Dump exported_program to `cwd/repro.py`, with run command. )minifyrun checkpointsTexist_okr))r$save_dirr&config_patchesrepro.pywr*)r$r/r.r&module_in_commentNzWriting repro file to %sNo write permissions for %s)ospathjoinrexistsmakedirsioStringIOsave_graph_repro_epr getvaluegetcwdopenlogwarninguse_buckrwriteOSError) r$r%r&r'subdiroutcurdir file_namefds rdump_to_minifyrI@s5 ' '' ' WW\\,.- 8F 77>>& ! FT*(kkm  -"  &clln55GGLL4  Bi% #!%5#*#!&*  KK2I > +113   B KK5y A Bs* E)D;<=E;EEE)(E)cBd}tj|}||S)Nc |jd}t|dk(rd|zS|jd}tt|D](}||}|j dk7r d|z||<$d||<*dj |}|dz|z}|S)N z# r)splitlenpoprangestripr6)s_sfirstilines r_convert_to_commentz.get_module_string.._convert_to_commentss HHTN q6Q;"9 as1v AQ4Dzz|r!d{!!   IIaL DL1 r)rconvert)gmrY module_strings rget_module_stringr]rs$ %,,R0M } --rFr*) r$r[argsr/ stable_outputr.r&accuracy check_strr2strictr[r^r/c | | td| | td| | td|+|J|Jtjj||| }n||j}t |} |j | |j t ||||| d|v} |j d|j d|j d | d |d |d | d y)Nz.One of exported_program and gm must be definedz2Only one of exported_program and gm can be definedz-If gm is defined, args should also be definedrbr'r_r. _accuracyzif __name__ == '__main__': z3 from torch._dynamo.repro.aoti import run_repro zf with torch.no_grad(): run_repro(exported_program, config_patches=config_patches, accuracy=z , command=z , save_dir=z , check_str=z) )rtorchexportmoduler]rB(generate_compiler_repro_exported_program)rHr%r$r[r^r/r_r.r&r`rar2rbr\s rr;r;s/(BJ PQQ# TUU ~$, OPP~~ <<..r4.G   $ $ &&b)MHH]HH0 "'  -/HH +,HH CDHH WW_Vbblmtlwx<|I= =r)r/r`rbc ptjjtd}tjj |stj |dtjj|t |jjd}tjdt |jj|t|d5}t|||t||||d| dddtj} tjj| d} tj || tjd | t"rt%|j'yy#1swYxYw#t($rtjd | YywxYw) Nr+Tr,z.pyz&Writing checkpoint with %s nodes to %sr1)r[r^r/r.r`r2rbr0z(Copying repro file for convenience to %sr3)r4r5r6rr7r8rPgraphnodesr?r@r>r;tupler=shutilcopyfilerArrBrC) r[r^r%r/r`rbrDrGrHrF repro_paths rdump_compiler_graph_staterrsFWW\\,.- 8F 77>>& ! FT* VBHHNN(;' K  Y ' - - / #   & ? 1:>?sF:A FFF54F5rec ntjdt|dt|dtd}|s|dt j jdz }tt j dr!|dt j jdz }tt j d r!|d t j jd z }|tz }tjj|d }t jj!|||d |dz }|dz }|d|dz }|S)NrL)r_z5 import torch import torch._inductor.inductor_prims z! isolate_fails_code_str = None z z# torch version: cudaz# torch cuda version: git_versionz# torch git version: z zexported_program.pt2z&exported_program = torch.export.load('z') z # print(exported_program.graph) zconfig_patches=)textwrapdedentr r r rgversion __version__hasattrrtrurr4r5r6rhsave)r$r'r_r. model_strep_paths rrjrjs8  678m456   I ()B)B(C2FF 5==& ) 1%--2D2D1ERH HI 5==- 0 01J1J0K6R RI.00 ggll8%;    ! KK@""   !J j #:+;+; <!"H4@ ,  ! ; ! ;s ;"B00Ccdtjj_|j }|j \}}|||fS)NT)rg _inductorconfiggenerate_intermediate_hooksriexample_inputs)r'r$modr^kwargss r repro_commonr,s>9=EOO6  ! ! #C#22LD& f rc,t||\}}}|||fSN)r)r'r$r/rr^rs rrepro_get_argsr3s#$W.>?Cv f rcddlm}t||\}}}ddlm}||||d|j |d}|D]-} t | tjs| js+d}n|r|yy)Nr_aoti_compile_and_package_inner synchronizeT load_and_runcheck_accuracyinductor_configsF) torch._inductorrr torch.cudarr` isinstancergTensoris_cuda) r'r$r/rr[r^rr need_syncargs r repro_runr8s}?#G-=>Bf&#  ''' I c5<< (S[[I   rTreturncddlm}m} tjj |||}|j }|S#t $rx}|rYd}~yt||r|j|jk(rYd}~yt|tr&d}tj|t|Yd}~yt||d}~wwxYw)Nr) UserError UserErrorTyperdz/Found .* in output, which is not a known type\.)torch._dynamo.excrrrgrhri Exceptionr error_typeINVALID_OUTPUT RuntimeErrorresearchrr) r[ tuple_inputsrbskip_export_errorrrepepatterns rexport_for_aoti_minifierrSs;* \\ \& A YY[  *  a # 8T8T(T a &HGyy#a&)5") *s'3> B?B:%B:81B:. B::B?c j ddlm}ddlm ddlm}t |\}}}|||||\} d} jdvsJjdk(jdd l m d |D]-} t| tjs| js+d nd fd } |||tj | j" tj t$| |j&j(j*j,j.j0 y)Nr)minifierr)_aoti_flatten_inputs)r' aot_inductor)dynamopythonrrFTc"t|}t|| }|yt|tjj sJ ||dj r y#t$r}||t|vrYd}~yYd}~yd}~wwxYw)N)rbrFTr) rnrrrgfx GraphModuler`rrepr) r[flat_example_inputsrarrrrrr'rrbrs r module_failsz"repro_minify..module_failss01 %  V?P  :"ehh22333  +!&//!1    $$q')A s A'' B0B  B)ra)r%r/r`rb)r dump_stater.offload_to_disk skip_offload skip_sanitymax_granularityr)functorch.compilerrrtorch._inductor.compile_fxrrminifier_export_moderrrrrgrr functoolspartialrarrr`r.rskip_saving_eager_intermediatesrr)r'r$r/rrrr^rrr%rrrrrrrbrs` @@@@@@r repro_minifyrws**??$W.>?Cv-A T6>-))#M  ' '+? ?? ?  ) )X 5F11&I c5<< (S[[I  <  &&|w?P?PQ$$ %')%%  !!//<<''//rrNr)r/r&r`r. tracing_moderarrr`c "| D]} tjd| durdndurdtjd|d|dd |d d |d tj } fd} | j ddd} | j dd}| || j dd}| || j dd}| ||jddd|jddd|jddd |jd!tdd"#|jd$t|d%#|jd&t|d'#|jd(t|d)#| j d*}| ||jd$t|d%#d}ttjd+kr|gtjd+d}| j|}tt t"d,}||j$|||-S).NzPUnrecognized kwarg %s; perhaps this repro was made on a newer version of PyTorchTr`FrNzAn AOTI repro script, typically triggering a bug in PyTorch AOTInductor. When run with no arguments, this script defaults to running 'z8'. Extra flags may be available; to find out more, try 'zr --help'. There are also alternate subcommands available, see below. default settings on this script: accuracy=z tracing_mode=z save_dir=z check_str=rL) descriptionformatter_classc|j}|jddddd|jdddd |jd ddd d |jd tdd|jdddddy)Nz --no-accuracyr` store_constrNz>do not test accuracy, just run the module and see if it errors)destactionconstdefaulthelpz --accuracyatest if the RMSE between the compiled module and the fp64 reference is greater than eager and the fp64 reference. This is usually more reliable than the standard allclose test, as we expect numeric differences from compiling, often improving accuracy over eager. RMSE test allows for compiled module to diverge greatly from eager, as long as this divergence moves it closer to the 'true' mathematical value of the network. Caveats: (1) double precision can still suffer from rounding error, so it is not a perfect reference (see for example 'Herbie: Automatically Improving Floating Point Accuracy') for approaches that detect the necessary working precision and compute it in arbitrary precision floating point; unfortunately, this is not practical for tensor computation; (2) if there are not enough samples in the output being compared, we may get unlucky and have an unlucky greater RMSE than eager; this could be overcome by applying a more rigorous statistical test at some p-value, which we leave for future work. )rrrrz--strict-accuracystrict_accuracyaby default, when doing accuracy minification we will reject reductions which change the divergence from a floating point divergence to a integral/boolean divergence. This is because some operations like ReLU involve temporarily sharp boundaries that smooth out again afterwards; without requiring divergence on floating point, the minifier will often fixate on divergent boolean tensor even though this is not the true source of the divergence. However, rejecting these reductions makes it more difficult for the minifier to make process. Using this option will let the minifier progress for ALL divergences--you just might not end up with a useful repro in the end.z --save-dirDIRz!directory where saved inputs live)typermetavarrz --no-save-dirr.z(don't use any directory for saved inputs)rrrr)add_mutually_exclusive_group add_argumentr)parseraccuracy_groupr`r.s r common_flagszrun_repro..common_flagss<<>##  Q $  ##   $ , ##  # J $ $  4     ;  rr&z {run,minify})rrrequiredr*zjust run the repro)rr)zrun the minifier on the reproget_argsz get the argsz!--skip-saving-eager-intermediates store_truez+skip saving eager intermediates on --minify)rrz--offload-to-diskzYduring minification, offload delta debugging intermediates to disk. Use if you're OOMingz --skip-sanityz@skip sanity check at beginning of minification on original graphz--max-granularityz;start at this granularity and work down; must be power of 2)rrrz --check-strzBrequire minified program to fail with error containing this stringz--minifier-export-modezThe export mode used in minifier, either dynamo or python.`dynamo` corresponds to strict=True, and `python` corresponds to strict=False.z--skip-export-errorz1Skip intermediate graphs that cannot be exported.zminifier-queryrM)r)r*r)r/)r?r@argparseArgumentParserRawTextHelpFormatteradd_subparsers add_parserrintrboolrPsysargv parse_argsrrrr&)r$r/r&r`r.rrarr more_kwargskrr subparsers parser_run parser_minifyparser_get_argsparser_minifier_queryr^r' COMMAND_FNSs `` r run_reprors  ^   4 U   $ $>>EYG66=Y?  +/  + , !55F ? B&& 'J&& !'J))6*M ++J^+LO!+ :  h  O   J    Q   $ ]  ! @ '11&'&&  Q ' D 388}'#((12,'%G"K (;w '!. r)r)N)FT):__doc__rrr9loggingr4rrorrv importlibrtypingrrrrgtorch._dynamo.debug_utilsrrr r r r r rrr torch.exportr torch.hubr getLoggerr r?inductor_config is_fbcoderArrrdictrIr]nnModulernr;rrrjrrrrrrrrrrrs"  #''    )g! 89  $ $ &5 5(, /B%/B/B/Bd38n % /Bd.037$(!%/3    7/ 7 ! 7 5:  7T#s(^,7~  $?^)-  #d38n %#L4 87;! ehhoo!HGZ04 !# !|T#s(^,| D#I |r