Vh=!0sUddlZddlZddlZddlZddlZddlmZddlmZddlm Z m Z ddlm Z m Z ddl mZmZmZmZmZddlZddlZddlmZddlmZddlmcmZddlmZm Z m!Z!ddl"m#Z#dd l$m%Z%dd lm&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/dd l0m1Z2dd l3m4Z4ejjjlZ6gZ7e8e9e:d<ejvjxjzZ=GddeZ> ddedej~de@fdZAe eAej~jdZCe eAej~jZDe eAej~jZFde!deGde!fdZHe#e=je/deDde!de!fd ZIe#e=je/deDde!de!fd!ZJe#e=je/deDde!de!d"eLd#eLfd$ZKe#e=je/deDd%e!d&eLd'eLd(eLd)e@d*e!f d+ZMe#e=jjgd,ZPe#e=jjBgd-e!fd.ZQe#e=je/eDd/e!de!fd0ZRe#e=je/deDd%e!d/e!fd1ZSe#e=je/dd%e!d/e!d2eLd3eLfd4ZTe#e=je/eDd/e!de!fd5ZUe#e=je/eDd%e!d/e!de!fd6ZVe#e=je/dd%e!d/e!d#eLfd7ZWe#e=je/deDd%e!d/e!d8eLd9e@fd:ZXe#e=je/deDdd;e!d/e!de!fd?ZZe#e=je/eDd/e!de!fd@Z[e#e=je/deDd%e!d/e!de!fdAZ\e#e=jd/e!dBe!de!fdCZ]e#e=jd%e!d/e!dBe!de_e!e!ffdDZ^e#e=je/eDd%e!d/e!dEe!dFeLdGeLdHe@d9e@de!fdIZ`e#e=je/deDd%e!d/e!dJe!de!fdKZadLe!dMeGfdNZbdOejfdPZde#e=je/eDe>jjfd/e!dQe!dMeGde!fdRZee#e=je/deDd%e!d>e!dQe!dMeGfdSZhe#e=jddTZje#e=je/eDe>jjdUfd/e!dQe!dMeGd"eLfdVZke#e=jjeDd%e!d/e!dQe!dMeGd"eLf dWZle#e=jjeDd%e!d/e!dQe!dMeGd"eLde!f dXZoe#e=jjeDd%e!d/e!dQe!dMeGdYeLf dZZpe#e=jjeDd%e!d/e!dQe!dMeGdYeLde!f d[Zrd%e!d/e!dQe!dBee!dMeGd\eGd]e!de!fd^Zse#e=je/deDd%e!d/e!deGde!fd_Zte#e=je/dd%e!d/e!dQe!dBee!dMeGd\eGd]e!de!fd`Zue#e=je/dd%e!d/e!dQe!dBee!dMeGd\eGd]e!de!fdaZve#e=je/eDde>jjfd/e!dQe!dBee!dMeGde!f dbZwe#e=je/deDde>jjfd%e!d/e!dQe!dBee!dMeGde!f dcZxe#e=je/eDe>jjfd>e!dQe!dMeGde!fddZye#e=je/deDe>jjfd%e!d/e!dQe!dMeGde!f deZze#e=je/dd>e!dfe!dgeLfdhZ{e#e=je/die!dje!de!fdkZ|e#e=je/d%e!dle8eGdeGdmeGdneGdoeGf dpZ}e#e=jjB dd/e!deGdmeeGdneeGdoeGf drZde!deGdmeeGdneeGde_eGeGff dsZe#e=je/ dd>e!dte!deGdmeeGdneeGdoeGf duZe#e=je/d%e!dle8eGdeGdveGfdwZe#e=je/d%e!dle8eGdxeGdyeGdzeGf d{Zd%e!de!d|ejfd}Ze#e=j e/deCd%e!d~e!deGd|ejfdZe#e=j e/eCd%e!d~e!deGd|ejfdZdZe#e=je/d>e!de8eGde8eGde8eGde8eGde!f dZe#e=je/eDd>e!de8eGde8eGde8eGde8eGde8eGde!fdZe#e=je/d%e!de!d'eLfdZe#e=je/d;e!de8eGdeGdeGdoeGde!f dZe#e=jjeD dd%e!d/e!deeLde!fdZe#e=je=jjje6je=jjje6j d>e!dgeLdee@fdZe#e=j"e/ddd>e!dgeLdee@fdZe#e=j$e/de!deGde@fdZe#e=j&e/dde!deGde@fdZe#e=j(e/ ddBe!de!deGde@de@de!f dZe#e=j*e/d%e!de!deGdeGde@f dZde8eGfdZde8e!deGdeGde8e!fdZde8e!fdZde8e!deGfdZde8e!deGdeGfdZe#e=j6je=j6jg dde8e!deGdeGdee!de!f dZe#e=j8je=j8jg dd/e!de8eGdeGdee8e!dee8e!f dZe#e=j:jBdd>e!deGdeGde_e!dffdZe#e=j<j dd>e!de8eGdeGde_e!dffdZe#e=j>jBdd/e!deGdeGde_e!dffdZe=j@jBje6j dd/e!de!deGde_e!dffdZe#e=jFe/deDdd/e!de!de!d"eGd&eGf dZe#e=jHe/eD dd/e!de!de!d"eGd&eGde@f dZe#e=jJe/eDdd/e!de!de!d"eGd&eGf dZe#e=jLjeDd%e!d>e!de!de!dee!deGdeGdeGdeGde8e@de_ee!ee!ee!ffd„Ze#e=jLjd%e!d>e!de!de!dee!deGdeGdeGdeGde8e@dejBdejBdejBde_ee!ee!ee!ffdĄZdee!dee!fdńZe#e=jRjde!d>e!de8eGde!de!dBee!dee!de8e@de_ee!ee!ee!ffdɄZe#e=jRjde!d>e!de8eGde!de!dBee!dee!de8e@dejBdejBdejBde_ee!ee!ee!ffdʄZd>e!dBee!dee!dee!dee!dHe@deLdeLde@de_e!e!e!ee!ee!ffdτZe#e=jXe/dddѫd>e!dBee!dee!dee!dee!dHe@deLdeLde_e!e!e!ffd҄Ze=jXjje6j e=jXjje6jd>e!dBee!dee!dee!dee!dHe@deLdeLde_e!e!e!ffdӄZe=j\jje6jdde8e!fdԄZe#e=j`jd>e!dBee!dee!de!de!deLdeLde_e!e!e!ffdՄZe#e=jbjd>e!dBee!dee!de!de!dHe@deLdeLde_e!e!e!ffdքZe#e=jbjdd>e!dBee!dee!dHe@deLdeLde_e!e!e!ffdׄZe#e=jhjd>e!dBee!dee!de!de!dHe@deLdeLde_e!e!e!e!e!ffd؄Zd>e!dBee!dee!de!de!deLdHe@de!fdلZe#e=jljd>e!dBee!dee!de!de!deLdeLde_e!e!e!e!ffdڄZe#e=jnjd>e!dBee!dee!de!de!deLdeLde_e!e!e!e!e!e!ffdۄZe#e=jpjd>e!dBee!dee!de!de!deLdeLde_e!e!e!e!ffd܄Ze#e=jre/ddeDdd݄Ze#e=jve/dddddddޜdee!e'fdOeejdeejxde@de@deejzf dZe#e=j|e=j~e=jge/dZe=jjje6j e#e=je/ddddd>e!dBe!dee!dee!dee!dHe@deLdeLfdZdZe#e=jjde!d>e!dBee!dee!dee!dee!dee!de@deLde8e@de!de_e!ee!ee!ffdZe#e=jjde!d>e!dBee!dee!dee!dee!dee!de@deLde8e@de_e!ee!ee!ffdZe#e=jjde!d>e!dBee!dee!dee!dee!dee!de@deLde8e@dejBdejBdejBde_e!ee!ee!ffdZe#e=je/dddëd>e!d%e!dBe!dee!dee!dee!dee!deLfdZe#e=je/dddëd>e!d%e!dBe!dee!dee!dee!dee!deLde!fdZe#e=je/eDd>e!de_eGeGffdZd/e)de)de8eGdeGfdZe#e=je/d/e)de)de8eGfdZe#e=je/d>e)de)de8eGde8eGde8eGf dZe#e=jdqdde)deGdve)de)d&e'f dZe#e=je/dqdde)deGdve)de)d&e'f dZdqdde)deGdve)de)de@d&e'f dZe#e=jje=jjje6jddZe#e=jde)deGdve)de)fdZe#e=je/de)deGdve)de)fdZde)deGdve)de)de@f dZe#e=je/d~dJeDd/e!de_e!e!ffdZe#e=je/ dde!dee@eGeLfdee@eGeLfdeejfdZe#e=jddZؐdZِdZe#e=jje#e=jje#e=jje=jjje6je=jjje6j e=jjje6je=jjje6j e=jjje6je=jjje6j d>e!dee8eGdee8eLde!fd Ze#e=jje#e=jje#e=jje=jjje6je=jjje6j e=jjje6je=jjje6j e=jjje6je=jjje6j d>e!dee8eGdee8eLde!fd Zdd Ze#e=jje=jjge=jjje6je=jjje6j e/dd  dd>e!de8eGd eeLde!fdZe#e=jje=jjge=jjje6je=jjje6j e/dd  dd>e!de8eGd eeLde!fdZe#e=jje=jjge=jjje6je=jjje6j e/dd  dd>e!de8eGdeeLdeeLde!f dZe#e=jje=jjge=jjje6je=jjje6j e/dd  dd>e!de8eGdeeLdeeLde!f dZe#e=jje=jjge=jjje6je=jjje6j e/dd  dd>e!de8eGdeeLdeeLdeeLde!f dZe#e=jje=jjge=jjje6je=jjje6j e/dd  dd>e!de8eGdeeLdeeLdeeLde!f dZeD dd>e!de8eGd e8eeLde@de!f dZdZdZdZdZ ddZdZdZdd Zdd!Zd"Ze#e=jje=jjje6je=jjje6j d#Ze#e=jje=jjje6je=jjje6j d$Ze#e=jje=jjje6je=jjje6j d%Ze#e=jje=jjje6je=jjje6j d&Zd'Zdd(Zdd)Zd*Ze#e=jje=jjje6je=jjje6j d+Ze#e=jje=jjje6je=jjje6j d,Zd-Zd.Ze#e=jje=jjje6je=jjje6j d/Ze#e=jje=jjje6je=jjje6j d0Ze#e=j je=j jje6je=j jje6j d1Ze#e=jje=jjje6je=jjje6j d2Ze#e=jje#e=jje=jjje6je=jjje6j e=jjje6je=jjje6j e=jjje6je=jjje6j d3Z e#e=jje=jjge/ dd>e!de8eGd4e@deeLde!f d5Z e#e=jje=jjge=jjje6j e/ dd>e!de8eGd4e@deeLdeeLde!f d6Z e#e=jje=jjge/ dd>e!de8eGd4e@deeLdeeLdeeLde!fd7Z dd8Z d9Zdtee!d:ee!d;e!de!fd<Zd:e*de!fd=ZeDd>e!de8eGd4e@d e8eeLde!f d>Ze#e=j$jd?e!d@e!de@fdAZe#e=j&e=j(ge/dBZe#e=j*gdCZe#e=j,gddDZe#e=j.gdEZe#e=j0gdFZd/e!dQe!dBee!dMeGd\eGde_e!e!ff dGZe#e=j4e/d~d]d/e!dQe!dBee!dMeGd\eGde_e!e!ff dHZe#e=j6e/d~d]d/e!dQe!dBee!dMeGd\eGde_e!e!ff dIZde!dJeLde!fdKZde!dJeLde!fdLZdMe!de*fdNZdOe*dPe!de!fdQZdPee!de!fdRZ dSeGd4e@dOejdejxfdTZ!dUe!dVeGdWeGd4e@fdXZ"dUe!dYeGdVeGdWeGd4e@f dZZ#dUe!de8eGd4e@fd[Z$dUe!de8eGd4e@fd\Z%e#e=jLe/eDdUe!de8eGd4e@fd]Z& dd?e!d^e!d_eGd`eGd4e@dae@de!fdbZ'e#e=jPe/eD dd?e!d^e!d_eGd`eGd4e@de!f dcZ(e#e=jRe/eDddZ)e#e=jTe/dde>jjfdeZ*dfejBdgejBdhe@de@fdiZ+e=jXjje6je=jXjje6je/djddkdlZ,e#e=jZje=jZjge=jZjje6j e/eD dd>e!de_eGeGfd4e@dmeeLdneeLde!f doZ.e#e=jZje=jZjje6je=jZjje6j e/eD dd?e!dee_eGeGfd4e@dee_eLeLfde!f dpZ/e#e=j`e#e=jbe#e=jdeDe/d?e!de_eGdfde!fdqZ3e#e=jhe#e=jje#e=jleDe/d?e!de_eGdfde!fdrZ7d?e!de_eGdfdseeGeGeGge!fde!fdtZ8e#e=jre#e=jte#e=jve/dduZ<e#e=jze/dvdwdddxdyZ=e#e=j|e/dddzd{Z>e#e=j~je=j~jge/dejddd|dne'dOeejd}ejdeejxde@f d~ZBe#e=j~jgdejddd|dme'dne'dOeejd}ejdeejxde@f dZDe#e%dZEe#e=je=jjje6j e/dqdqde>jjfd>e!dQe!dge'de'dBee!dMeGde!fdZFe#e=je=jjje6j e/d~dd>e!dQe!dMeGde_e!e!ffdZGe#e=jj ddddde!de!d-e!deLde@dee!d'eeLde_e!e!ffdZIdZJe#e=jge/eDddZKe#e=je/dZLe#e=jdZMe#e=jje=jjgdddd/e!dOeejdee!de!fdZOe#e=jje=jjgdd/e!deeGfdZRe#ejxjzjddZSe#e=je/ddddZTe#e=jjddd/ejBdeejdejBfdZUdddZVddddZWe#e=je/dZXe#e=jddZYeJe=je=jeJe=je=jFeJe=je=jJeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jeJe=je=jy(N)Iterable)Enum)partialreduce)chainproduct)AnyCallablecastOptionalUnion) sym_floatsym_intTensorregister_decomposition) out_dtype)IntLike NumberTypesuggest_memory_format TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_out out_wrapper)_pytree)tree_map__all__ceZdZdZdZdZy) ReductionrN)__name__ __module__ __qualname__NONEMEANSUML/home/dcms/DCMS/lib/python3.12/site-packages/torch/_decomp/decompositions.pyr!r!0s D D Cr+r!Fftype_promotioncompute_dtype_onlycJtjfd}|S)Nctj|i|Dcgc]}t|ts|}}t j |d i\fd}fd} t ||it ||} r|St ||Scc}w)Ntype_promotion_kindcJt|tr|jS|SN isinstancerto)xcomputation_dtypes r, increase_precz0type_casts..inner..increase_precHs"!V$tt-..r+cJt|tr|jS|Sr4r5)r8 result_dtypes r, decrease_precz0type_casts..inner..decrease_precNs!!V$ttL))r+)pytreearg_tree_leavesr6rutilselementwise_dtypesr) argskwargsr8 flat_argsr:r=rr9r<r/r-r.s @@r,innerztype_casts..inner>s--t>v> *QPVBWA  +0*B*B + ,:+ '<     x t, P0O P HM1- -1 s BB) functoolswraps)r-r.r/rFs``` r, type_castsrI9s' __Q..6 Lr+T)r.r/)r.r8dimreturncjt||jz D]}|jd}|SN)rangerJ unsqueeze)r8rJ_s r,_unsqueeze_to_dimrRks2 3= ! KKO Hr+ grad_inputout_gradyc4|d||zz jzSNr" conj_physicalrTrUs r, tanh_backwardr[qs  q1q5y//1 11r+c4||d|z zjzSrWrXrZs r,sigmoid_backwardr]xs  qAE{113 33r+beta thresholdct||zj}tj||z|kD|||z|dzz SN?)exptorchwhere)rTr8r^r_zs r,softplus_backwardrgs= TA ;;DI-xAS9Q RRr+ grad_outputalphascale input_scale is_resultself_or_resultc||z}|}|}|r&tj|dk||z||zz||zStj|dk||z|ztj||zz||zSNr)rdrerc) rhrirjrkrlrmnegcoefposcoef negiptcoefs r, elu_backwardrssemGGJ{{ a  * $(@ A ' !  {{ a  * $w .>J;V1W W ' !  r+c.tj||Sr4)rd full_likeselfvalues r, fill_scalarrys ??4 ''r+rxctjjdk(fdtj |S)Nrc,djdS)Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrJ)rxsr,zfill_tensor..sRSXS\S\S^R__jkr+)rd_checkrJatencopyrvs `r, fill_tensorrs3 LL qk 99T5 !!r+rwcftjtj|dzdddz SNrminmaxrdclamprws r, hardsigmoidrs) ;;u{{4!83 ;a ??r+cHtj|dkD|dkz|dzdS)Ngg@gUUUUUU?rdrerhrws r,hardsigmoid_backwardrs0 ;; $y!  r+min_valmax_valcBtj||k||k\zd|S)Nrr)rhrwrrs r,hardtanh_backwardrs$ ;;DGO?M r+c6tj||kd|Sror)rhrwr_s r,threshold_backwardrs ;;ty(![ 99r+negative_slopeself_is_resultc<tj|dkD|||zSror)rhrwrrs r,leaky_relu_backwardrs ;;taxkN.J KKr+grad approximatecd}d}d}|dk(ri||zdz}d}||z}||z} |||| zzz} tj| } d|z} d| z} d| z}d| | zz }|dd|z|zzz}| |z|z}|||zzS|}||zdz}ddtj||zzz}|tj||zd zz}||||zzzS) Ng;f?g;f?gmBP ?tanhrgHm?r"rg)rdrerfrc)rrwrM_SQRT2 M_SQRT1_2 M_2_SQRTPIkBetakKappax_sqx_cuberF tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs r, gelu_backwardrs'%G&I'Jf*$s*d{/0ZZ& TzJ+j:55 AF T(9$9:/14DD)99::Y&,Q4&=112eiit d 233sTCZ'((r+inputctjtj|}tj|}||zd||zz z}|||zzSrW)rdrFsoftplussigmoid)rhrinput_tanh_softplus input_sigmoidouts r, mish_backwardr sV **QZZ%67MM%(M - 1':=P'P#P QC -3 44r+c2|tj|zSr4)rdrrs r,silurs %--% %%r+c\ddtj| zz }||zd|d|z zzzSrW)rdrc)rhrwrs r, silu_backwardrs<1uyy$''(G  AG (<$< ==r+weightc<tj|dkD|||zSror)rwrs r, _prelu_kernelr%s ;;taxv} 55r+c~tj|dkD|||z}tj|dkDd||z}||fS)Nrrr)rhrwr input_grad weight_grads r,_prelu_kernel_backwardr*sE TAX{F[4HIJ++dQhTK-?@K  $$r+noiseloweruppertrainingcx|r||z dkDr|j|S||zdz }tj||||S)Ngư>r#)mulrr)rhrwrrrrrrs r,rrelu_with_noise_backwardr5sKEEMD(u%%%-1,'' ~~  r+bufferc|dk}tj|dd}tj|dd}tjtj| }||||d|zz zz zS)Nrr"rN)rdrercabs)rhrwr in_negative max_derivsignrfs r,log_sigmoid_backwardrJsg(K KA.I ;;{Ar *D 599T?"#A )da1q5k&:: ;;r+loss reductionc|tjjk(rtj|S|tj jk(rtj |S|Sr4)r!r(rxrdmeanr)sum)rrs r,apply_loss_reductionrWsHINN(((zz$ imm)) )yy r+dtypec|tjk(rtjS|tjk(rtjS|tj k(rtj Syr4)rd complex32float16 complex64float32 complex128float64rs r, to_real_dtyper`sK }} %// !}} %"" "}} #r+targetc*||z dz}t||S)Nr#)r)rwrrrs r,mse_lossros 6Ma D i 00r+c||tjjk(rd|jz nd}|||z z|zS)N@)r!r(rxnumel)rhrrrnorms r,mse_loss_backwardrys; #,y~~/C/C"C3 D 56> "[ 00r+ctj|||}|jtd}tj||d}tj |}tj |||S)N)rJrz-infTrJkeepdim)rdsoftmaxeqfloatall zeros_likere)rwrJrrmasked masked_rowszeross r, safe_softmaxrs[ --#U 3C WWU6] #F))FT:K   S !E ;;{E3 //r+rbc||z j}tj||kd|dzz|z |d|zz }t||S)Nrr#)rrdrer)rwrrr^rs r,smooth_l1_lossrsO 6M   D ;;td{C$'MD$8$t:K LD i 00r+c|tjjk(rd|jz nd}||z }t j |}||z}t j ||k||z|z |t j|zSra)r!r(rxrrdrrer) rhrwrrr^rr8abs_x norm_grads r,smooth_l1_loss_backwardrs{ "+inn.B.B!B3 D v A IIaLE{"I ;;  A EJJqM! r+cht|||||}t||jt||dSNT copy_fromcopy_to exact_dtype)rrshaper)rhrwrrr^rSresults r,smooth_l1_loss_backward_outrs3%[$ 4 PFj&,,/ FJD QQr+deltac |tjjk(rd|jz nd}||z }t j || k| |z|zt j ||kD||z|z||z|zSra)r!r(rxrrdre)rhrwrrrrr8s r,huber_loss_backwardrs "+inn.B.B!B3 D v A ;; UF  e# AItk1E94!8k;QR r+cht|||||}t||jt||dSr)rrr r)rhrwrrrrSr s r,huber_loss_backward_outrs3!dFIu MFj&,,/ FJD QQr+ ignore_index total_weightc|jdkrdnd}|tjjk(r||z }|j |}t j ||k7|d}t j|} t j| ||d} | j|jcxkDrdkDrnn|j |}|Nt|jD cgc]} d} } |jd| |<|j| }||z}t j ||k7|d}| |zScc} w)Nr#rr"g) rJr!r(rxrPrdrerscatterrOr reshape) rhrwrrrrr channel_dim safe_targetrSrQ new_shapes r,_nll_loss_backwardrsxxzA~!1KINN(((!L0   k *F++f 4fa@K!!$'Jz; TJJ~~+//+/a/!++K8   %dhhj 121Q2 2!'a + *!F* ++f 4k1EK  ##3s* D<c|jdkDsJdtj|j|}|j|}|dzdk(s Jd|d||dz}|j |d|}|j |||}t j |}d|z |z|z|z} ||z}t j|| g|S)Nrz*glu does not support 0-dimensional tensorsr#z.Halving dimension must be even, but dimension z is size rbr|)rJr@canonicalize_dimsizenarrowrdrcat) rhrwrJwrap_dimnIn inputSize firstHalf secondHalfgradInputFirstHalfgradInputSecondHalfs r, glu_backwardr)s 88:>GGG>%%dhhj#6H ))H C 7a< 8 )C5Q<qI Ha3IXy)H MMr+c d|jcxkr dksJdJd|jdksJd|jdk(xr|jdk(}|sA|jd|jdk(s"Jd|jd|jd|jdk(s'Jd |jd |jd f|'|j|jd k(sJd |tjj k(r}|jdk(rj|jdk(r|jd|jdk(srJd|jdd|jd|jd|jdkr|jdk(sJd|jt |||||||S)Nrr#input tensor should be 1D or 2Dr";0D or 1D target tensor expected, multi-target not supportedsize mismatch (got input: , target: ):expected total_weight to be a single element tensor, got: z (z elements)rNzV\\^tzz"~=F =INN(((TXXZ1_ A%+*;*;A*>$**Q-*O EdjjQRm_U)oo/00G HYHYZ[H\G] _ O  A%+*;*;*=*B EkFWFWEX Y B T669lL r+c >|jdk(sJd|j|jdk(sJd|j|jd|jdk(r>|jd|jdk(r|jd|jdk(s!Jd|jd |j|jdk(s&Jd |jd |jd t|||||||S) NzSonly batches of spatial inputs supported (4D tensors), but got input of dimension: rzUonly batches of spatial targets supported (3D tensors) but got targets of dimension: rr#r"r-r.r0z ( z , elements))rJr rr)rhrwrrrrrs r,nll_loss2d_backwardr58s@ 88:? ]^b^f^f^h]ij? ::<1  _`f`j`j`l_mn  1 a( JJqMV\\!_ , JJqMV\\!_ ,I $DJJqsgG D6M *U[[T9JPW-X XF &INN((($**,& Mr+crtjtj| |z}t||Sr4)rdr8rcr)rrrrs r,soft_margin_lossr@s. ;;uyy%&1 2D i 00r+c||ztj||zdz z}|tjjk(r||j z }|SrW)rdrr!r(rxr)rhrwrrrSs r,soft_margin_loss_backwardrBsM+%v})E)IJJINN((($**,. r+otherpc6tj||z |S)N)rD)rr)rrCrDs r,distrFs 99UU]a9 ((r+x1x2c|jdjdd}tj|tj}|jdjdd}tj|tj}tj |j d||gd}tj |||gd}|j|j}|jdjS)Nr#rNT memory_formatr) powrrd ones_likecontiguous_formatr!rmatmulmT clamp_minsqrt) rGrHx1_normx1_padx2_normx2_padx1_x2_r s r,_euclidean_distrZsffQimmB%G __WE4K4K LFffQimmB%G __WE4K4K LF ))RVVBZ&12 6C ))R)2 .C ZZ F   A  # # %%r+ input_sizesstartendstepcX|j|}tj||||||Sr4) new_zerosrd slice_scatter)rhr[rJr\r]r^rSs r,slice_backwardrbs/&&{3J   z;UC NNr+r"cddlm}m}|j}|dk(r t dt j |j|}t|j}t|j} |dkr t d||nd} ||ntj} || dkr| ||z } || dkr| ||z } || dkrd} n|| ||kDr||} || | kr| } n,|| tjk(s|| ||kDr||} |j| | |zz} | | z } | |zdz |z||<| |xx|zcc<|jr td|j|| | S)Nr)guard_size_obliviousstatically_known_truez,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver"z>r+cn|j|dtffd }||dd}|||}||fS)zn Normalize start and end such that both are in the range [0, x.get_size()[dim]] and start <= end. rKcL||S|dkr|z}tt|||Srorr)valrrdefaultdim_sizes r, clamp_wrapz(_normalize_start_end..clamp_wraps0 ;N 7.C3sE?E**r+r)r int)r8rJr\r]r}r|s @r,_normalize_start_endrsJwws|H+#+ ua1 -E S%8 4C #:r+srcc |tj|j|}|j|}t ||||\}}t |j}||z |dz z|z||<|j |}|dk(r||k(r|dk(r|jSdg|jz}tj||j} | |z |z||<tj||jtj} |dk7rtj| | |k\} ||k7rtj| | |k} |dk7rtj| | |z |zdk(} dg|jz} d| |<| j| } t j#| t j%|| |d|S)Nr"rdevicerrrN)r@rrpr rrhexpandclonerJrdarangeronesbool logical_andviewrre_unsafe_masked_index) rrrJr\r]r^r|src_sizeindicesidxmask mask_shapes r,raras  S 1C{{3H%eS%=JE3EKK H5[D1H-$6HSM **X C zcXo$!)yy{fuyy{"G ,,x 5C%KD(GCL ::hu||5:: FD z  se|4 h  sSy1 qy  e t';q'@Auyy{"JJsO 99Z D ::dD55c4!Le TTr+indexcT|j|}tj||||Sr4)r`rdselect_scatter)rhr[rJrrSs r,select_backwardr:s+&&{3J    Ke DDr+offsetdim1dim2cV|j|}tj|||||Sr4)r`rddiagonal_scatter)rhr[rrrrSs r,diagonal_backwardrAs- &&{3J  ! !*k64 NNr+ input_dtypecF|j|k7r|j|}|Sr4)rr7)rhrSrs r,_cast_grad_to_input_dtyperJs&K']];/ r+outputc~||z}||tj||dzz }t|||jSNTr)rdrr contiguous)rhrrJrnew_grad_outputrSs r,_softmax_backward_datarRsK "F*O 6EIIS$-$J %[*k J U U WWr+c~|tj|tj||dzz }t|||Sr)rdrcrr)rhrrJrrSs r,_log_softmax_backward_datardsA uyy0599d4 J %[*k JJr+c||dzz||dz zz }ttjtj|}|d||j d}|d||z|j d} || zS)z/Utility function to implement im2col and col2imr#r"rrrrN)rrdrint64rP) input_dkernel_d dilation_d padding_dstride_drblocks_d arange_kwblocks_d_indices kernel_grids r, _im2col_col2im_indices_along_dimrpsQ&x!|)DDH EKKGI!Hh7AA!DAx*4jAKKBOK k ))r+ kernel_sizedilationpaddingric tjtdk(dtjtdk(dtjtdk(dtjtdk(ddd}|d|d|d d |d |jt}tj|d vxrt dddDfdt dt ddDtjt dDfd|dk(}|s|jd}|j\}} } } \} } \}}\}}\}}t| |||| |j}t| |||| |j}tj|||||f}|jdjd}|dddd||f}|jdddddd}|jd}|jd}|j|| |z|z||z}|s|jd}|S)Nr#cy)Nz"im2col(): only 2D kernel supportedr*r*r+r,r}zim2col..r+cy)Nz$im2col(): only 2D dilation supportedr*r*r+r,r}zim2col..rr+cy)Nz#im2col(): only 2D padding supportedr*r*r+r,r}zim2col..rr+cy)Nz"im2col(): only 2D stride supportedr*r*r+r,r}zim2col..rr+c~|rtd|Dntd|D}tj|dy)Nc3&K|] }|dkD ywrNr*.0rDs r, z1im2col..check_positive..(Q1q5(c3&K|] }|dk\ ywrr*rs r,rz1im2col..check_positive..;RqAF;Rrcy)Nz<{param_name} should be greater {'than' zero, but got {param}r*r*r+r,r}z0im2col..check_positive..rr+rrdr~param param_namestrictconds r,check_positivezim2col..check_positives3,2s(%((;RE;R8R X r+rrrFrrirr4c3&K|] }|dk7 ywrr*rds r,rzim2col..:!qAv:rrc dtS)NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tupler sr,r}zim2col..--25\N<r+c3\K|]$\}}}}}d|d|zz||dz zz dz |zz&yw)r"r#Nr*)rrpaddilkersts r,rzim2col..sF "Cc3 S1s7]SC!G_ ,q 0R 77s*,rLc3&K|] }|dkD ywrr*)rcs r,rzim2col..s 'aAE 'rc Fdtdddddddd S) Nz!Given an input with spacial size rL, kernel_size= , dilation= , padding= , stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r)rr output_sizerr risr,r}zim2col..sL3E%*4E3FG"m;xj9)9VH-]D Fr+r4rrNr"rT)rdr~rur rrziprPrrrrpermuterrsqueeze)rrrrrirrp batched_input batch_dimrinput_hinput_wstride_hstride_w padding_h padding_w dilation_h dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indices padded_inputrnum_blocks_rownum_blocks_colrr s ```` @@r,im2colrs~ LL[!Q&(TU LLX!#%ST LLW"$QR LLV!#OP ; .8Z(8Yu568$ KKE u:D LL :3:uRSz:: < &) "#J;' K  LL '; '' F FAIM "/4{{,I{GWHh"Iy%J $Hh9:y(ELL::y(ELL 55Iy) LML+55b9CCBG !Q 24FF GF ^^Aq!Q1 -F',,Q/N',,Q/N ^^;)H4n~6UF " Mr+rc !"tjtdk(dtjtdk(dtjtdk(dtjtdk(dtjtdk(ddd}|d |d |d d |d|d|j"t"}tj|dvxrt d"ddD"fdddz}tj"d|zdk("fdt D cgc]"\} } } } } d| d| zz| | dz zz dz | zz$}} } } } } |d|dz!tj"d!k(!"fdtj!dkD!"fd|dk(}|s|j d}|j"\}}\}}\}}\}}\}}|j"d"d|zgz|z}|jdddddd}t||||||j}t|d}t||||||j}t Dcgc] \}}|d|zz}}}|j"d"dtzg|z}dd||f} tj|| |d}t!j"|| | | | f}|s|j%d}|Scc} } } } } wcc}}w)Nr#cy)Nzonly 2D output_size supportedr*r*r+r,r}zcol2im..rr+cy)Nzonly 2D kernel supportedr*r*r+r,r}zcol2im..rr+cy)Nzonly 2D dilation supportedr*r*r+r,r}zcol2im..rr+cy)Nzonly 2D padding supportedr*r*r+r,r}zcol2im..rr+cy)Nzonly 2D stride supportedr*r*r+r,r}zcol2im..rr+Tc~|rtd|Dntd|D}tj|dy)Nc3&K|] }|dkD ywrr*rs r,rz1col2im..check_positive..rrc3&K|] }|dk\ ywrr*rs r,rz1col2im..check_positive..rrcy)Nz9{param_name} should be greater than zero, but got {param}r*r*r+r,r}z0col2im..check_positive..rr+rrs r,rzcol2im..check_positives3,2s(%((;RE;R8R U r+rrrFrrir)r#rc3&K|] }|dk7 ywrr*rs r,rzcol2im..rrrLc dtS)NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: rrsr,r}zcol2im..rr+rr"cdddS)Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rLz and kernel_size=r*)rr sr,r}zcol2im..s#==B2YKH"m%r+rNc:ddddddddd S NzGiven output_size=rrrrz , expected input.size(-1) to be but got rN.r*Lrrrrr risr,r}zcol2im.. F$[M }M:Zy &B))*9U2YKqBr+c:ddddddddd Sr r*rsr,r}zcol2im..rr+rr4r accumulater)rdr~rur rrrPrrrrrRr`prodr_unsafe_index_putrrr)#rrrrrrirrpprod_kernel_sizerrrrrcolrout_hout_wrrrrrrrr indices_row indices_colorDoutput_padded_sizerrrr s# ````` @@r,col2imrs LL[!Q&(OP LL[!Q&(JK LLX!#%IJ LLW"$GH LLV!#EF ; .8Z(7Ie468$; . KKE u:D LL :3:uRSz:: < #1~ A6 LL b $$) %'* (K'    "Cc3 S1s7]SC!G_ ,q 0R 77 C  AQA LL b Q B B  LL A B B AIM " KKELE5Hh"Iy%J $Hh MM58U1X1A%AB[PSVV WE MM!Q1a +E2 xY%,,K$K3K2 xY%,,K14K0IJ1!a!e)JJ __ q58tK0014FFF {K 0C  # #FC4 # HF UU6YJ YJ K LF " Mk VKs 6'M 9Mrcz||j||zzjtj|}|SNrJ)type_asrr@r)rhrrjrEs r,native_dropout_backwardr#8sB  [1E9 :AA11+> B A Hr+ input_size dimensionrct|dk(rtj|dStjt||}tj |||j tj}|jd||j}|jd|dzj||dz}|j|}d|z|fz}tj|||djS)NrrrNr"r4Tr)rurd squeeze_copyr@rrrint32unfoldflattenmovedimr`rrr) rr$r%rr^rJrrSrs r,unfold_backwardr,Gs  :!!!$**  Z) r9r< unnormalizedx_maxr s r,_softmaxrHs Aww%**$$$&+&>&> uDDLL'#|  AwwyA~yy|  1c40yyU+ EIIlCF FF <( Mr+)r c|j}|r|jtjk(sJt j |tj j\}}|j|}|jdk(r|}ntj||d}||z }tjtjtj||d}||z }|s|j|}|Sr@)rrrdrBr@rArCrDr7rrEr:rrc) r8rJr>r9r<shiftedrGshifted_logsumexpr s r, _log_softmaxrLs Aww%**$$$&+&>&> uDDLL'#|  AwwyA~ 1c40e) %))EIIg,>T"RS ( (F <( Mr+r padding_idxscale_grad_by_freqsparsec|jdk(sJd|jdkr4|jd|}|jdk(r|jd}|S||S)Nr#z'weight' must be 2-Dr"r)rJrp index_selectr)rrrMrNrOrs r, embeddingrRsd ::<1 444 ||q!!!W- <<1 ++a.C gr+ num_weightscttj|tjj\}}|j |}t |t j}|rZ|j|f}t j|}tj||g|d}||} || jdz }t||k(|j} |j| d} |j|f|j |jdz} tj| |g| dj |S)NrATrrNr)r@rArCrDr7rrdlongr`rNrrrPrRrp masked_fillr ) rhrrSrMrNr9r<countsrgrad_weights_scalerr grad_weights r,embedding_dense_backwardrZs,',&>&>)N)N)V)V'#|..!23K%guzz:G""K>2w''' 4D'Q#G_!$6$@$@$DD W 3[5E5E FD  " "4 +D'' **7<<>::K  ! !+y$4 ! P S S r+c"d}|D]}||z} |SrWr*)r8rEis r,rrs$ A  Q Hr+tensors num_chunkscZg}|D]}|j}|||zdz |z|z}|||k7r;dgdz|j|z dz zd|||z gz}tj||d}|d|t j |dgz}|j |j||S)Nr"rr#rN)rrprconstant_pad_ndrdSizeappendr) r]rJr^padded_tensorstensor tensor_size pad_along_dimr view_sizes r, _pad_chunkrhs N 6kkm $S)J6:zIJV K, ,#'V[[3.23 C 007C))&#q9F% J3C(DD fkk)45 6 r+cR|dj}|D]}|j|k7syy)NrFTrp)r]rprds r,have_same_ndimsrk s2 1:??D ;;$  r+c|djd|}|D]-}tj|jd||k(d/y)Nrcy)NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr*r*r+r,r}z+leading_dimension_matches..rr+)rrdr~)r]rJleading_dim_sizesrds r,leading_dimension_matchesrosN )$3/  KKM$3 #4 4 ]  r+ctj|dk\dtjt|dkDd|dj}|dj}|D]r}tj|j dkDdtj|j|k(dtj|j|k(dtt |r(tj|dj|}nEtj|dk\d|D]&}tj||jkd (t|||S) Nr"cy)Nz&_chunk_cat expects positive num_chunksr*r*r+r,r}z._preprocess_chunk_cat_inputs..rr+rcy)Nz0_chunk_cat expects a non-empty input tensor listr*r*r+r,r}z._preprocess_chunk_cat_inputs..!rr+cy)Nz#_chunk_cat expects non-empty tensorr*r*r+r,r}z._preprocess_chunk_cat_inputs..&rr+cy)Nz8_chunk_cat expects all input tensors with the same dtyper*r*r+r,r}z._preprocess_chunk_cat_inputs..)rr+cy)Nz8_chunk_cat expects all inputs tensors on the same devicer*r*r+r,r}z._preprocess_chunk_cat_inputs..-rr+cy)NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr*r*r+r,r}z._preprocess_chunk_cat_inputs..4rr+cy)Nz3_chunk_cat expects dim < ndim for all input tensorsr*r*r+r,r}z._preprocess_chunk_cat_inputs..9rr+) rdr~rurrrrkr@rrJrpro)r]rJr^expected_dtypeexpected_devicerds r,_preprocess_chunk_cat_inputsrzs-  LLq"RS LL G qTQZ%%Naj''O   V\\^a')VW LLN * N   MM_ , N  w$$WQZ^^%5s; 1H a  F LLfkk!M   gs+ Jr+rct|||}t|||}|tj||dzStj||dz||S)Nr")r)rzrhrdr!)r]rJr^rrcs r, _chunk_catr|?sS 'wZ @Cj9N {yyq11 .#'s3 r+ split_sizesctj|||}|.|Dcgc]"}|jtj$c}St ||D])\}}t ||jt||d+ycc}w)Nr|rJTr) rsplit_with_sizesrrdrOrrr r)rwr}rJrsplitssrsplits r,split_with_sizes_copyrPs " "4# " >F {HNO1e&=&=>OO f- NMFE fekk 2 UF M N Ps'B split_size.cDtjj|||Sr4)rrr)rrrJs r, unsafe_splitrcs ::  UJ 44r+cDtjj|||Sr4)rrr{)rr}rJs r,unsafe_split_with_sizesrhs  ( ( S AAr+c|j}||}|dk(r|dk(sJ|jfS||zdz |z}ddlm}||}t |Dcgc]}|}}|||z|z z |d<t j |||Scc}w)Nrr") guard_intrN)r detachrfrrOrdr) rwrrJr[r|chunksrr\r}s r,rros**K3HQ1}} #a'J 6F@ v F',V}5!:5K5 J$7($BCKO ;;t[# ..6s Btensor_indices_or_sectionsc|jjdk(sJ|jtjk(sJ|j tj dk(xsdk(fddk(r4|j}t|tsJ|j||S|Dcgc]}|j}}|j||Scc}w)Ncpur"rcddS)Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr*) split_dimsr,r}zAtensor_split_tensor_indices_or_sections_py_impl..s< ! 99S>r+vecc|js&|jst|}t|}|tj||z}|dk(r|S|j dk(r||zS|||zzSro)r:rr~rdmvr)rwrrr^rirs r,addmvrss  ! ! #DOO,=4yE  %((4% %C qy  yy{ad{  r+rrstdgammaNCHxWgroup output_maskc Rtj|||dtj||dtj|dtj|j zzk(fdtjj fk(fdtjduxsj k(fdt\} } tj| dk(fdtj||jjdg } |jjdg } d}d}d}| dr*d | zz }tj| jdj| jd}tj| jdj| jd}tj|jd jd | }n| j| jd}| j| jd}tj|jd tjd | f|j }|z|z |z|z|z|z}| z||z|zz }|jd }t|d}t|d}tj|j| |tj|j| |z|z}|j|j j!|j"}| d rk| j| | j| jd zz |jd zjdg j}| dr| jdg }|||fS)NF)allow_cpu_scalar_tensorscdzzdS)NzExpect input to have z elementsr*)rrrsr,r}z,native_group_norm_backward..s'A }I>r+c.dddjS)NzExpect mean to have shape (, z , but got r)rrrsr,r}z,native_group_norm_backward..s-aS5'DJJ<Pr+c<ddjSdS)NzExpect gamma to have z elements but got rN)r)rrsr,r}z,native_group_norm_backward..s-'s*.s,QC/[\a[bcr+r#r|rbrNr"rr4)r@check_same_devicecheck_same_shaperdr~rr divmodrrrrPrrrrRr7r)rhrrrrrrrrrcpg_remdsdbd_inputd_gammad_biasrds_valdb_valc1c2c3s ` ````` r,native_group_norm_backwardrs UD$ 5+N 4F LL Q$> LL q%j P LL  +!+i q% IC LL  c ; & + +Aq# 6 : :s : CB   !Q $ ( (aS ( 1B $G $G#F1~ 39   YYr5??1#56>>q%MQQRSTFYYr5??1#56>>q%MQQRSTFr" a,B ZZ5#.2215FZZ5#.2215Fr" Auc?4;;?Btmf$ ,t 3d :Q >S4Z&4-!+ + \\"  r1 % r1 % IIk))!UC=r Bii aS92> ?   //%++.11%++>1~E3'"''!UC*@4>>RTCU*UU..$%SaSS\ WQZ 1~QC Wf %%r+out2c  t|||||||||| } | | | f}t| D]2\}}| t|||jt |||d4|Sr)r enumeraterr r)rhrrrrrrrrrr7r8rr rSr\rEs r,native_group_norm_backward_outr4sz"(UD$q!S%Fd#J&!Q1 = jmQWW 5 Q 1 4 PQ r+c,||j|S|Sr4r7)r8rs r, _maybe_castrQs}ttE{ Hr+grad_outnormalized_shapebiasc8"|j}|j} tj|j""fd||||fD\} } } } | J| t |z }||d}|d|}g}g}t | D]*}||k\r|j||j|,t|}t|}ddl m }||dks ||dkrN|dr|j|nd|dr|j||dnd|dr|j||dfSdfSt|| j}t|| j}| |z |z}| | | z}n| }||z}tj||d}tj||}tj||d}tj||}||z |z }d}d} d}!|dr||z |z}|dr0| .t |dkDrtj| |z|d} n| |z} |dr8| 6t |dkDrtj| |d}!n| j!}!t#||jt#| |jt#|!|jfS)Nc3hK|])}|!|jtjn|+ywr!)r7rdrOrr8r9s r,rz-native_layer_norm_backward..fs>9  =  e.E.EF  9s/2rrdr"r#TF)r rJr@get_computation_dtyperrurOrbrrfrdr`rRrdrrrr)#rrrrrrrr input_shape input_ndim grad_out_cast input_cast weight_cast bias_castaxis inner_dims outer_dimsinner_dim_indicesouter_dim_indicesr\rMrdx_hat grad_x_hatabrrrrFrd_weightrr9s# @r,native_layer_norm_backwardrXs++KJ33EKK@9E640 95M:{I  $$ $ ,- -DTU#JUd#J#%#% : ( 9  $ $Q '  $ $Q ' ( ZA ZAJAF#';AF'C,7NEOOK (3>q>EOOK. /t3>q>EOOK. /  HL  T:>>#3 4D T:>>#3 4D $ $ &E"[0 " QA */6A :u %B 2($ /B 5" B EBJE $G!%H#F1~!8u$1~+1  !A %yy!68I5QH$u,H1~)/  !A %YY}.?GF"((*F GU[[)Hekk*FEKK( r+c t||||||||} || | f} t| D]2\} }| t| | |jt || | d4| Sr)rrrr r)rrrrrrrrr7r8rr rSr\rEs r,native_layer_norm_backward_outrsw(%)4vt[Fd#J&!Q1 = jmQWW 5 Q 1 4 PQ r+ running_mean running_varmomentum functionalc 2dgttd|jz} tj|j } |} |} |r"tj|j } |j | } tj| | dd\}}tj||z}||z |z}tj|| }tj|| }|!||zd|z |zz} |s|j| |2|j|jdz }tj|| }|||dz z z}||zd|z |zz} |s|j| n||J|j | d}|} |j | d}|} |}dtj||zz }|jj dk7r|}|}n"|j#d }|j#d }t%||jdz }t%||jdz }||z |z}|2|j'}t%||jdz }||z}|2|j'}t%||jdz }||z}|jj dk(r8|j |j }|j |j }|j |j ||| | fS) Nrr#rT)rJ correctionrr")rrrr)rhrOrJr@rrr7rdvar_meanrsqrtrcopy_rr rSrrr`rRr*)rrrrrrrr-rreduction_dimsr9new_running_meannew_running_var input_acc biased_varrrr save_mean save_rstdn squeezed_var unbiased_varinvstds r,native_batch_norm_helperrsS4a 566N33EKK@#!O!77 DHH#4H5  >> >a  D{{:+,$,$&MM$7 MM$7  #')3q8||6SS ""#34  "  A.A!==^DL'1A;7L&5X8TTO!!/2'K,CCC#->TJ '!nn+<4nH %ejjs!234 <<   %$II-I-I uyy{Q7"6599;?;$,&( !"6599;?;& ||~ uyy{Q7$ ||E!LLu{{L3 LLu{{L3   $  r+r save_invstdc >t||||||||d \}} } } } || | fSNFr rrrrrrrr-rrrrQs r,native_batch_normrs=*B vt\;(CQV*&FIy!Q 9i ''r+c ||tj||||||S| td| td|rtj||||||||Stj|||||||S)Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r_native_batch_norm_legitrg$_native_batch_norm_legit_no_training)rrrrrrrr-s r,native_batch_norm_decompositionr4s 3,, 648S   <   <  ,, 64{HhPS  88 64{Hc  r+cl|j|}||zdz |z}|dk(rZ|dk(rU|Dcgc]}|}}|||z|z z ||dz <tjjjj |||Stjjj j|||Scc}wNr"r)rrdopsrrr{rr)rdrrJr|rrQr}s r,unsafe_chunk_py_implr Ys{{3HV#a'F2JQ8q=+12az2 2", V0Ch0N"O FQJyy~~55==fkSVWW 99>> & & - -fj# FF3s B1c Ntjj|||||d||Sr)rrr{)rrrrrrr-s r,rres5  ( ( 0 0      r+c >t||||||||d \}} } } } || | fSrrrs r,rr{s=*B vt\;(CQV*&FIy!Q 9i ''r+c >t|||dd|||d \}}}} } |||fSrr) rrrrrr-rrrrQs r,!_native_batch_norm_legit_no_statsrs<*B vtT48S%*&FIy!Q 9i ''r+c ft||||||||d \}} } } } | Jd| Jd|| | | | fS)NT#new_running_mean should not be None"new_running_var should not be Noner) rrrrrrrr-rrrrrs r,#_native_batch_norm_legit_functionalrsl" ! vt\;(CQU    'N)NN '  &L(LL & 9i)9? JJr+c Ttjj|||||d|}d}|tjjjk(r tjj ||}tj |tj|j|jS)a Return a reserve tensor for batch norm, used only by cudnn to pass forward state to the backward pass. This is needed for `_batch_norm_with_update` and `_batch_norm_no_update`, which support a variety of backends including cudnn. We create this tensor here to get the correct shape in the traced graph if we detect that will call the cudnn kernel, and rely on DCE to avoid materializing this tensor. Tr)rlayoutr) rd_C_select_batch_norm_backend_BatchNormBackendCudnn(_get_cudnn_batch_norm_reserve_space_sizeemptyuint8rr) rrrrrr-rbackend reserve_sizes r,_get_batch_norm_reserve_tensorrs hh11 vt\;cGL%((,,222xxHH 8  ;;EKK U\\ r+c dt|||||d||d \}}} } } t||||||d} ||| | fS)NTFrrr rrrrrrr-rrrrQreserves r,_batch_norm_with_updater%sa*B      *&FIy!Q- vt\;dG 9i 00r+c t|||||d||d \}}} } } t||||||d} | Jd| Jd||| | | | fS)NTr!rrr") rrrrrrr-rrrnew_rmnew_rvr$s r,"_batch_norm_with_update_functionalr)s ! vt\;hT  - vt\;dG  DDD   CCC  Iy'66 BBr+c dt|||||d||d \}}} } } t||||||d} ||| | fS)NFr!r"r#s r,_batch_norm_no_updater+sa*B      *&FIy!Q- vt\;eG 9i 00r+c|Jtj||kjtj}|j ||zd|z z}||fS)Nrrb)rdr;r7rr")rrD generatorrr=s r,_fused_dropout_decompositionr."s[    OOE "Q & * * * =D ,,u  %q 1C ;r+)rrr pin_memory non_blockingrKrr/r0rKc|r|tjk(sJd|rJdt|tjtt t tfsJ|0|.|,t|tjr|jS|Sd}t|tjr|}ntj|}|c||jk7rT|1|jdk(r"tjj||}d}tjj|||}|$|s"tjj||}d}|tj||S|S)NTODOFrTrJ)rdstridedr6rr~rrcomplexr scalar_tensorrr_primsconvert_element_type device_put) r8rrrr/r0rKdtype_convertedx_tensors r,_to_copyr;,s25==08&8 0!6!> a%,,UD'B CC C ~%-M,A a &779 HO!U\\"&&q) f7  !5||885IH"O<<**8V\J <<44XuE {{8=AA Or+c,tj|Sr4)ralias)r8s r,nop_decompositionr>Zs ::a=r+out3exponential_average_factorepsilonc tj||||||||\}} } |r%|| | |jdtjfS||jd|jd|jdtjfS)Nrr)rrr`rdr) rrrrrrr@rArrrs r,cudnn_batch_normrCbs$$  " GAq!1aU[[ABB  EKK0  r+ct|D]>\}}|dk(s ||jkr|j||k(r.|j|}@|SrW)rrpr rP)r8broadcast_maskrrs r,_broadcast_batch_norm_backwardrFsO/" d 19dQVVm 0E D!A" Hr+r$c *t|||||||||| Sr4)native_batch_norm_backward) rrrrrrrr3r-rr$s r,batch_norm_backwardrIs/ &     r+c &|j} | |j} n| } tj|j&&fd|||||||fD\} } }}}}}|j}|j }|dk\sJdd}t t |||z }|}|}|r||"J||J|}tj||z}dg|z}||||<g}t|D]}||k7s |j|t||}d|z }tj| |}tj| | |z z|}t||z|}ttj||z||z|} |t||dz}!nt||z|}!|r| |z | z}"| |"z |z |!z}#n| |!z}#| dr||z}$nd}$| dr|}%nd}%|#j| t|$| t|%| fS)Nc3HK|]}||jn|ywr4rrs r,rz-native_batch_norm_backward..s,  $%= a7 s"r#z$rank of the input must be at least 2r"rb)rr@rr rJrrhrdrrOrbrFrrr7r)'rrrrrrrr3r-rr weight_dtyperrrrunning_mean_castrunning_var_castsave_mean_castsave_invstd_castr input_rankr num_featuresrrrEreduction_axesr\rgrad_output_sumdot_p grad_mean proj_scale grad_scaleprojrSrY grad_biasr9s' @r,rHrHsx++K || " 33EKK@          ++KJ ?BBB? D[)*[->>L D F F$666!,1A1MMM -34!"j 0N&t,N4 "N : % 9  ! !! $% *$ ?D  Dii ~>O IImzD'89> JE./E~VI/ %$,0J 3FNKcQ 3 [ .  T!Z/$t+y8JF "Z/ 1~fn  1~#    k"K.I|, r+c  t|||||||||| } | | | f}t| D]2\}}| t|||jt |||d4|Sr)rHrrr r)rrrrrrrr3r-rr7r8rr rSr\rEs r,native_batch_norm_backward_outr\ s"(    Fd#J&!Q1 = jmQWW 5 Q 1 4 PQ r+save_varc Btj|||||||d|gd SNT)TTTrrH)rrhrrrrr]rAs r,miopen_batch_norm_backwardra6 s5  * *    r+ reserveSpacec Btj|||||||d|gd Sr_r`) rrhrrrrr]rArbs r,cudnn_batch_norm_backwardrdP s5  * *    r+c|j|jttjdvfd|jddD]}tj|dk7fd d|dzdk(rxd|dzdk(rjt dt dd|D}t dt dd||D}tjjj|||Sd d fd }|d|d\}}}} |d|d\} } } } |d t|d | f}| s| stj|dSd}||||| d\}}||| | | d\}} d}tt|jdt|jdD]!\}}| |d |dd|f}||d |dd|fz}#||| zz S)NrcdS)Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r*rjsr,r}z%adaptive_avg_pool2d..u sKD6Rr+rLrc"dtdS)Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape rrrsr,r}z%adaptive_avg_pool2d..z s99>uaIr+rNc3,K|] \}}||zywr4r*)rr\rs r,rz&adaptive_avg_pool2d.. sG$!QqAvGc3:K|]\}}}||dz |zz ywr"Nr*)rr\rrs r,rz&adaptive_avg_pool2d.. s' '1aAQ! O sc8tj||z|dS)Ntrunc rounding_moderddivrrrs r, start_indexz(adaptive_avg_pool2d..start_index syyQ99r+cJtj|dz|z|zdz |dS)Nr"rmrnrprrs r, end_indexz&adaptive_avg_pool2d..end_index s&yy!a%1q1,awGGr+ctj| tj}|||}||zdz}||z}|dk(xs||zdk( }|r|dz }n |dk(r|dz}tj| tj}|jd|z}|rUtj|dz |j |j } tj|| } |||} | |z } n|} || ||fS)Nrr"rrNr)rdrrrPr5rrminimum)in_sizeout_sizeorangei0 maxlength in_size_modadaptive range_maxrmaxvali1lengthrrurss r, compute_idxz(adaptive_avg_pool2d..compute_idx s hvU[[I 7 3x'!+ ( #q(GH{,Ba,GH  NI A  NILL6M ll2* ((! 399SZZF--V,C68W5B"WFFFIx//r+.r4)rrNr|ct|tr||fS|dksJ||jdk\}|dk(r t|d}t j ||d}t|| }||fS)NrrNrLr4r)r6rrPrRrdrV)valsrrr~rJrs r, maybe_maskz'adaptive_avg_pool2d..maybe_mask sw fg &< 7N7 0 0 44Dby(q1$$T45D&vt4F< r+)r~rJr)rr rurdr~rrnnr avg_pool2drRrrrO)rrrrikernelridxhlength_h range_max_h adaptive_hidxwlength_w range_max_w adaptive_wrrretr\jrrurpr rss @@@@@r,adaptive_avg_pool2drk s. \\F KKE u:D LL R[[   F I   Ry;r?"a'E"I B,G1,LG#eBCj+*FGG +.uRSz;+O  xx""--eVVDD:H0@/:%)[QS_.U+D(K.9%)[QS_.U+D(K 'a0$6 7D jzz$H--    h jbND(  h jbND( Cdjjn-uTZZ^/DE+1 ;sAq!|$CS!Q\**C + (X% &&r+ctjd|dttj|j d| }ttj|}dg|j z}|j d| |d| |tj||jj||zzjd}|jt|j d| t|z}tj|jd|g|jddj|j S)N max_unpooling d_forward_outr"rrNFr)r@alert_not_deterministicroperatorrr rprrrrrr`rhr) rwrrrJnchwindices_nc_shape indices_flatrs r, _max_unpoolndr s !!M#m"DE  djj3$/ 0B  k *BsTYY"jj3$/Usd$++b+5::;KLrQQ gbk^^DEcT!23d;6GG HF  ! !r\NDLL,< "  d6<<r+ctjjtjk(fdtjt dk(fdtjj dvfdtjj j k(fdtdj D].tjjdkDfd 0tdS) Nc"djS)Nz2elements in indices should be type int64 but got: r)rsr,r}zmax_unpool2d.. sDW]]OTr+r#c"dtdS)NzMThere should be exactly two elements (height, width) in output_size, but got elements.rursr,r}zmax_unpool2d.. ;'(  4r+rc$djdS)NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with dimensions.rjrsr,r}zmax_unpool2d.. s%%)YYK| =r+c<djdjSNzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: r)rrwsr,r}zmax_unpool2d.. s,PQUQ[Q[P\]229-- Br+r"rc*djddS)NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got  with dimension being empty.r)r\rwsr,r}zmax_unpool2d.. s%::,&6qcHr+) rdr~rrrurpr rOrr)rwrrr\s```@r, max_unpool2dr s LL $T LL KA  LL V  LL gmm# 1dii   IIaL1     w Q 77r+c tjjtjk(dtjjdvfdtjt dk(fdtjt dk(fdtjt dk(fdtjj j k(fdtd jD].tjjd kDfd 0tjd d kDxrd d kDxrd d kDfd tdS)Ncy)Nz(elements in indices should be type int64r*r*r+r,r}zmax_unpool3d..% rr+r4rc$djdS)NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with rrjrsr,r}zmax_unpool3d..) s^_d_i_i^jjvwr+rc"dtdS)NzVThere should be exactly three elements (depth, height, width) in output_size, but got rrrsr,r}zmax_unpool3d..- rr+c"dtdS)NzRThere should be exactly three elements (depth, height, width) in stride, but got: rrrisr,r}zmax_unpool3d..4 sdehioepdqq{|r+c"dtdS)NzSThere should be exactly three elements (depth, height, width) in padding, but got: rr)rsr,r}zmax_unpool3d..8 sefijqfress}~r+c<djdjSrr)rrsr,r}zmax_unpool3d..< s,PQVQ\Q\P]^229-- Br+r"rc*djddS)NzZmax_unpooling3d(): Expected input to have non-zero size for non-batch dimensions, but got rrr)r\rsr,r}zmax_unpool3d..E s% ;;-'7s-Ir+r#cdS)Nz5strides should be greater than zero, but got stride: r*rsr,r}zmax_unpool3d..N sGxPr+) rdr~rrrprur rOrr)rrrrirr\s`````@r, max_unpool3dr sD LL $&X LL fw LL KA  LL F q| LL G ~ LL w}}$ 1ejj !  JJqMA     LLq A 9&)a-9F1IMP a 88r+)rirdc$t||||d|S)NTinplaceri _index_addr8rJrrdris r, index_add_rT s aeVT GGr+c$t||||d|S)NFrrrs r, index_addr` s aeVU% HHr+rc8 tj|jtjjdkfdjdk(rj dnd |jdkDr|j nd tj k( fddk7rftj |j tj tk(xstjt  fd|z}|jdk(}|r|jdn|}dzfz}|rtjntj} | |||d} |r|S|r| jdS| j!S) Nr"c$djdSNz(Index should have dimension 1 or 0 (got r/rjrsr,r}z_index_add..y :5::,aHr+rcdddS)NzNumber of indices (z') should be equal to tensor.size(dim) (z ), for dim=r*)rJ index_sizeresr,r}z_index_add.. s!%j\1XYdXeeqmplrsr+c(dtddS)Nzalpha argument of type z cannot be safely cast to type !)r)ri python_typesr,r}z_index_add.. s-d5k]:YZeYffghr+r4Tr)r@canonicalize_dimsrprdr~r dtype_to_typerris_weakly_lesser_typerrPr index_put_ index_putrr)r8rJrrdrrizero_dimrGrrrrrres `` ` @@@r,rrm sM  ! !!&&# .C LL aH#(**/AqJ&,kkAo&++c"1K LLz!s z))!''2  4  E**4; D h %vv{H#QB C-5( "C#*I BV 5C!)s{{1~?s~~/??r+c  tjt|dkDdt|}|dj}|dd}t d|D}|r||f}n||f}||z}|dj ||}dt|z} t |D]j} || } tj| | d|| jdz fz|} |rtj|| d| }Rtj|| d| }l|S)Nrcy)Nz#received an empty list of sequencesr*r*r+r,r}zpad_sequence.. rr+r"c3>K|]}|jdywrr)rr8s r,rzpad_sequence.. s/!&&)/s)rrrJr) rdr~rurrr9rOrr`r) sequences batch_first padding_valuesequences_sizemax_size trailing_dimsmax_lenout_dimsr dim_paddingsr\currseqrows r, pad_sequencer s# LLY!#%RS^N|  "HQRLM/Y//G"G,^,-'H A,  - 8CC ..L > "@A,"" \Q',,q/(A$BBM  %%c3AQ%?C%%c3AQ%?C@ Jr+c"t||||dS)NTr _index_copyr8rJrrds r, index_copy_r s q#ufd ;;r+c"t||||dS)NFrrrs r, index_copyr s q#ufe <. rr+rr4) r@rrprdr~rPrrrrr) r8rJrrdrrrGrrrs ` r,rr s  ! !!&&# .C LL aH vv{H#QB"'**/EOOA uE C-5( "C#*I BV $C!)s{{1~?s~~/??r+c*tj|jd|}tjtj| }|j s |j r|jd}n|}|tj|z |fS)Nr*r)rdrwr`rcrris_xpur8)rwrrfrs r,log_sigmoid_forwardr sn --r*D 1C 599T?"#A ||t{{% Q  ''r+lowhighr-ctj|jt|t||j|j |S)N)rrrrr-)prims_uniform_helperr rrr)r8rrr-s r,uniformr s=    cN t_ggxx  r+c<|jt||||Sr4)rr)rwrrr-s r,uniform_r s ::gdCy9 ::r+c"t|dz }|>tj|dudtjt||k(d|S|tj|dudtjt||k(dg}t|D]Z\}}t ||k(r$|j ||dzt |z8|j t ||dz|z\|Stjddy)Nr#cyNz9Must specify exactly one of output_size and scale_factorsr*r*r+r,r}z.upsample_compute_output_size.. rr+cyNr*r*r+r,r}z.upsample_compute_output_size.. rr+cyrr*r*r+r,r}z.upsample_compute_output_size.. rr+cyrr*r*r+r,r}z.upsample_compute_output_size.. rr+Fcyrr*r*r+r,r}z.upsample_compute_output_size.. rr+)rurdr~rr~rbr)r$r scale_factorsspatial_dimensionsr\rs r,upsample_compute_output_sizer s Z1, T ! O   S%);;ZH  4  O   S'+==zJ m, CDAq1v{"":a!e#4s1v#=>""7:a!e+>r+cg}t|}|rdnd}t|D]}||}|j| |z} || | | ||zz n| |z } tj|tj |j } | |z| zjtj} t|dz |z D]} | jd} |j| |S)Nrrrr"rN) rurOr rdrrrr7rrPrb)rrrrrnum_spatial_dimsrrr isizerjoutput_indices input_indicesrQs r,!_compute_upsample_nearest_indicesrB sG;'SsF # $&A --12/5ay/D*+%RW-e5==V(61U:>>u{{K '!+a/0 8A)33B7M 8}%+&, Nr+)preserve_memory_formatr rct|||gSr4r rrrs r,upsample_nearest1drb s UK& ::r+c"t|||gdSrrrs r,upsample_nearest_exact1dr n s UK& FFr+scales_hscales_wc t||||gSr4rrrr!r"s r,upsample_nearest2dr%| s UK(H1E FFr+c$t||||gdSrrr$s r,_upsample_nearest_exact2dr' s UK(H1ET RRr+scales_dc"t|||||gSr4rrrr(r!r"s r,upsample_nearest3dr+ s UK(Hh1O PPr+c&t|||||gdSrrr*s r,_upsample_nearest_exact3dr- s!  {Xx:$ r+rcDt||||}ddg|z}tj||}|jdk(rdt j |}|j d}|jjdk(r|dkrtj}|j|}|S)Nrr4r"cudarJ) rr _unsafe_indexrpr@rr rrrdrOr) rrrrspatial_indicesrr rK n_channelss r,r r  s8 {F%OTl_,G   w /F {{a33E: [[^ <<   &:>!33M"""? Mr+c |r|rd}n |rd}n|rd}nd}t||zdk(sJt|tdt||Dcgc]}t||||zc}Scc}w)Nrr4rr#r)rurOr)params has_biaseshas_projections group_sizer\s r, gather_paramsr8 s{o    v; #q (5#f+5 (38CK3T ./fQZ()  s A(c~|r'|d|z|d|z}}|d|zdz|d|zdz}}n||||}}d\}}||||fS)Nr#r"NNr*)r4hiddensr\ bidirectional cur_params cur_hidden bidir_params bidir_hiddens r,params_hiddensrA sk!'AAJ %+AEAI%6A 8Jl !'GAJJ %/" l z< ==r+c||kDsJ|j|jd|||z |jdd|Sro)rbr )r>last_batch_size batch_sizer;s r,update_hidden_for_packedrE sE Z '' ' NN:$$Q Oj4PQR   Q: ..r+c t||k(r|S||ksJtj||jd|||z fSro)rdconcatr )r>rCrD inp_hiddens r, update_hidden_for_packed_reverserI sP*$ Z '' ' <<    a*2N O  r+c X|d}|d}|r|dnd} |r|dnd} g} g} |r|dn|d} |jdd| }tj|t|}|r|ddd}|D]V}|jd}| |k(rn|rt || ||}nt || || }||||| || }|} | j|X|r| jn!| j|| jtj| d}|stj| dn|}||fS)Nrr"r#rrN) r rdrrhr rIrErbreverser!)inphiddenr4r5 hidden_fn batch_sizesrK ih_weight hh_weightih_biashh_bias step_outputr;rCr> split_inpr\r hidden_outs r,one_layer_rnn_datarW sRq Iq I%fQi4G%fQi4GK"$G)0k"ok!nOq!_5J Ck!23IddO ' IIaL a   9OQJ2OQJsJ 7IwW :&#'&z" ))K #C.57A&:J  ?r+cfd}|S)NcDtj||||zSr4rlinearr\r>rPrRrQrS nonlinearitys r,rFzrnn_cell..inner1 s AHHZGDqHIIr+r*r]rFs` r,rnn_cellr_0 sJ Lr+cfd}|S)Ncrtj|||}tj||||zSr4rZr\s r,rFzrnn_cell_data..inner8 s2 HHQ 7 +AHHZGDqHIIr+r*r^s` r, rnn_cell_datarb7 sJ Lr+c x|d}|d}|r|dnd}|r|dnd} tj|||} |r| jdn| } |jd} g} | D] } || | |||| } | j | "|r| j t j| d}|| jdfS)Nrr"r#r) rr[fliprPrbrKrdr!r)rLrMr4r5rNrKrPrQrRrSprecomputed_inputr>rTr\rs r, one_layer_rnnrf? sq Iq I%fQi4G%fQi4Gi95<)..q1BS!!!$JK 'q*i)WU :&' ))K #C  ""1% %%r+c|d}|d}|r |d}|d}nFtj|j}tj|j}|djd} |djd} g} d} | jd} d}d}d}d}|j }| j } | j } tj j jj|||||| | || | | |||||}|d|d|d}}}||jd|jdffS)Nrr"r#rF) rdrrrPrr rmkldnn_rnn_layerr{r)rLrMr4r5rKw0w1w2w3hxcxrOmode hidden_size num_layersr<rr3outputsrUhycys r,mkldnn_one_layer_lstmruU sS B B AY AY [[ # [[ #   Q B   Q BK D''!*KJMK E .. C B Biinn--55         !G$ GAJ 2rA rzz!}bjjm, ,,r+c |r|jddn|}g} t|D]} t||| |\} } }}|r || dz kr|nd}| || | |\}}| j||r!| ||||d\}}| j||r*t j |g|j dz }n|}|dk7s|s| |dz kst j||d}|r|jddn|}|| fS)Nrr"rT)rK)r3) transposerOrArbrdr!rJr6)rrMr4r5rqr6r3r<rlayer_fn final_hiddensr\r=r>r?r@fwd_inp fwd_hiddenbwd_inp bwd_hiddens r, _rnn_helperr~ s"&1EOOAq !eEM : >=K FA}> : J l$ QU(:'&uj*jQZ( "*|\:t# GZ   , IIw0'++-!2CDEE a,&1EOOAq !eE - r+c |jd} t||d}t|| |||||||ttt t j \} } | t j| dfSNrFrN) unbindr8r~rrfr_rdrstack rrmr4r5rqr6r3r<rrMrrys r,rnn_tanh_inputr tYYq\F 6:u 5F$   %**)=> C  M1- --r+c |jd} t||d}t|| |||||||ttt t j \} } | t j| dfSr) rr8r~rrfr_rdrrrs r,rnn_relu_inputr rr+c |jd} t||d}t|| ||||||dtt|t t j \} } | t j| dfSNrFrOrN) rr8r~rrWrbrdrr datarOrmr4r5rqr6r3r<rMrrys r, rnn_relu_datar {YYq\F 6:u 5F$    ##EJJ/ C  M1- --r+c |jd} t||d}t|| ||||||dtt|t t j \} } | t j| dfSr) rr8r~rrWrbrdrrrs r, rnn_tanh_datar rr+cltj||||z}|jd|}|dj} |dj} |dj } |dj} | |z| | zz} | | j z}||ntj||d}|| fS)Nr4rr"r#rrr[chunkrr)rLrmrnrQrS hr_weight chunk_dimgates chunked_gatesin_gate forget_gate cell_gateout_gatertrss r, lstm_cellr4 s HHRG ,s 2EKK9-MA&&(G"**,Ka %%'IQ'')H r Wy0 1B BGGI B ahhr9d&CB r6Mr+c (|d}|d}|r|dnd}|r|dnd}t|dk(r|dnt|dk(r|dnd} |djd} |djd} tj|||} |r| j dn| } g} | D](}t || | ||| d\} } | j | *|r| jtj| d}|| jd| jdffS)Nrr"r#rrr4r) rurPrr[rdrrbrKrdr!r)rLrMr4r5rKrPrQrRrSrrmrnrerTrs r,one_layer_lstmrB s*q Iq I%fQi4G%fQi4G[A%q F q8H6!9d   Q B   Q Bi95<)..q1BSK 3B 7IQRSB2 ))K #C A 1 . ..r+c |d}|d}|r|dnd}|r|dnd} t|dk(r|dnt|dk(r|dnd} g} g} |r|dn|d} tj|t|}|r|ddd}|d}|d}|j dd| |j dd| }}|D]}|j d}t j|||}|| kra| j|j d|| |z |j d|| |z f|j dd||j dd|}}|| kDrXtj||j d| || z fd}tj||j d| || z fd}t||||| | d\}}|} | j||r| j||f}nZ| j||f| jt| \}}tj|dtj|df}tj| d}||fS) Nrr"r#rrr4rNr)rurdrrhr r rr[rbrGrrKrr!)rLrMr4r5rOrKrPrQrRrSrrTr;rCrUorig_hxorig_cxrmrnr\rVhidden0hidden1rs r,one_layer_lstm_datar] syq Iq I%fQi4G%fQi4G[A%q F q8H6!9dKG)0k"ok!nO Ck!23IddO QiGQiGq!_-q!_- B  IIaLhhsIw/   NNIIaOa$78IIaOa$78  YYq!Q'1a);B  W^^AO8KLMqBW^^AO8KLMqB3B 7IQRSB236"X Bx =YYw*EIIgq,AA ))K #C  ?r+c4d}||||rtStS)a*Check whether we could use decompose lstm with mkldnn_rnn_layer. All the below conditions need to be met: * ``torch._C._get_mkldnn_enabled()`` returns ``True``. * All the input args are on CPU. * The dtypes of args are either torch.float or torch.bfloat16. * Inference. * ``has_projections`` returns ``False``. Args: * input: the input sequence to LSTM * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM * params: the weight and bias tensors of LSTM cNtjjsy|gt|ztt j |z}|Dchc]}|j }}t|dk7ry|j}|tj dk7ry|Dchc]}|j}}|D]&}|tjtjfvs&y|jry|djd|djdk7} | ryycc}wcc}w)NFr"rrr#T)rdr_get_mkldnn_enabledrhr from_iterablerrupoprrbfloat16 requires_gradr) rrmr4r]tdevicesrdtypesrr6s r, use_mkldnnz2select_one_layer_lstm_function..use_mkldnn sxx++-'DH$tE,?,?,G'HH%,-188-- w<1  U\\%( (#*+a!''++ EU[[%..99    Q%**Q-2a5::a=8 ).,s D#D")rur)rrmr4rs r,select_one_layer_lstm_functionr s!:%V$$$r+c t|dk(sJdt|||djd|djdk7}tt |d|d} t |||} t || |||||||| \} } tt | } | tj| ddtj| ddfS)Nr#lstm expects two hidden statesrr") rur8rrhrrr~rdr) rrmr4r5rqr6r3r<rrMrxrrys r, lstm_implr s r7a<999< 6:r!uzz!}1 1 /M NF #beRU# $F-eR@H$   Cm,-M  M!,a0%++mA>NPQ2R RRr+c t|dk(sJdt|||djd|djdk7}tt |d|d} t || ||||||dt t| \} } tt | } | tj| ddtj| ddfS)Nr#rrr"F)rO) rur8rrhrr~rrrdrrs r,lstm_data_implr s r7a<999< 6:r!uzz!}1 1 /M NF #beRU# $F$   #= Cm,-M  M!,a0%++mA>NPQ2R RRr+c&|jdd}tj|||jdd}|d|dzj}|d|dzj} |d|d|zzj } || z | z| zS)Nrr"r#r)rrr[rr rLr>rPrRrQrSchunked_igateschunked_hgates reset_gate input_gatenew_gates r,gru_cellrsYYq!_NXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH  !Z /( ::r+cPtj|||jdd}tj|||jdd}|d|dzj}|d|dzj} |d|d|zzj } || z | z| zS)Nrr"rr#rrs r, gru_cell_datarsXXc9g6<> 1 1 umWg r+ct|j||}t|d}t|d}tjj j |||||Sr)rrrrdr r_upsample_bicubic2d_aars r,upsample_bicubic2d_aa_vecrlsV ){M REmQ/GmQ/G 99>> 0 0 umWg r+czt|j||}|r|ndgt|z}t||||Sr4)rrru_upsample_linear)rrrrr rs r,_upsample_linear_vecrxs= ){M RE+]$#e*1DF E5- @@r+rc t||||gSr4r)rrrr"s r,upsample_linear1drs E; z JJr+c"t|||||gSr4r)rrrr!r"s r,upsample_bilinear2drs E; (?S TTr+c$t||||||gSr4r)rrrr(r!r"s r,upsample_trilinear3drs!  {MHh+I r+cL|r|dkDr |dz |dz z SdS| |dkDrd|z S||z S)Nr"rbrr*)rxryrrjs r,_compute_scalersB5=\# (S.1HqH#/EAIsU{U7XCUUr+c&|r||zS||dzzdz SNrr*)rj dst_indexrs r,_compute_source_indexrs$y   C(3..r+weightsweights_precisionctdt||Dd|dz zz}||z }tj|ddj tj S)Nc3K|]F\}}|jtj|jtjzHywr4)r7rdr()rrrs r,rz%_sum_tensors_uint8..s826!QU[[ADD--sA Ar"r) _sum_tensorsrrdrr7r)rrrrs r,_sum_tensors_uint8rsd:=c7:K  1$ %'F( (F ;;vq# & ) )%++ 66r+ctj|j}d}tj||j}d|d|dzzzz}|dk\}||j z S)Nrrr"i)rdrrrrr)r max_weightmax_weight_precision precisionsvaluesrs r,_compute_weight_precisionrsiW%))+J2:;L;LMJ :zA~!67 7F g D $((* ,,r+c jd}jdd}t|}tjtjj \}fd}t t|||D cgc]\} \} } }|| | ||dz | z } } } } }tt| \} }}g}tddgg|zD]c}ddgt|Dcgc]}||dk(r||n||c}z}tj|}t|}|j|ett|D]p} | | || z j!ddj#}t|ddd|dddDcgc]!\}}|t%j&||z |z#}}}rt|dk(sJ|d}tj(}j*j,dk(r|d krt$j.}t1|t$j2sJ|j5| }j7s|j9}|Scc}} } } wcc}wcc}}w) Nr"r#rAc|t|| |}tj| jj }t || j d}|j|jdgdg|z}|j tj}|dzj |dz }|||fS)Nrrrrrr"r) rrdrrr7rrrr r) inp_sizeryrnsqueeze scale_factorr\x_f32r8xp1rrrs r, get_valuesz$_upsample_linear..get_valuess%h-P  LL%,, 7 : : : G%lA}EKKPSKT ekk!n@sh/?@ HHU[[ !1umm1 m-a}r+rrrbr/rJ)r rur@rArC INT_TO_FLOATrrrhrrOrr0rrbreversedrr7rdrrrrrOr6rrr:round)rrrrr2 inp_sizesn_dimsrQrr\rryrxs_f32xsxp1svsrkrvxscalev1v2r rKrs` ` @r,rrs|QJ ABI ^F'' !AANNHAu 09  ; /0  +A+(F 8XvvzA~>F CL)FB B 1vh' (TluV}U!qtqybed1g=UU   uc * #Au - !  eFm $ )be#**3477>b1gr!$Q$x0 B 27F+ +    r7a<< UF//6M ||F"zB// fell ++ +   ]  ;F  " " $ MQV sI I' &I,rrc4|j|jk(Sr4r)rrs r, is_same_sizers 77agg r+c.tj||Sr4)rr)r8r rBs r,_reshape_aliasr !s 99Q r+c.tj||Sr4)rr)r8rs r,r0r0's ::a !!r+c2tj||||Sr4)rr)r8rrxrs r,rr,s >>!WeZ 88r+c|D]F}|tj|jtjtjfvdHtj|jtj k(dddlm}||jdk(r.7rr+cyNz*tensors used as masks must be bool tensorsr*r*r+r,r}z&_unsafe_masked_index..<rr+rrr"ry)rdr~rrUr~rrfrdr_meta_registrationsmeta_index_Tensorr9r rOrurrrr0rV)r8rrfillrrd meta_resultr\s r,rr1s   LL  EII66M  LL ejj < KAGGIN+//AA!WM zz+++T22 3w< ?   q A >GAJ?   a ) 5 5teT BBr+cL|D]F}|tj|jtjtjfvdHtj|jtj k(d|j dk(r|jStt|D]B}||}| |j|j| |j|dz ||<D|j|d}tj|||dS)Ncyrr*r*r+r,r}z5_unsafe_masked_index_put_accumulate..Srr+cyrr*r*r+r,r}z5_unsafe_masked_index_put_accumulate..Xrr+rr"ryTr)rdr~rrUr~rrrrOrurrrVrr)r8rrrrr\ masked_values r,#_unsafe_masked_index_put_accumulaterMs   LL  EII66M  LL ejj <  wwyA~wwy 3w< H   !&&)QGGAJH %%teQ/L  ! !!Wlt ! LLr+c|j}d}|dkrd}|6|dkDr*dg|z}|jd||<|j|}n|}||z}tj||k7|d} | j |} tj ||| j| } tj||k7| d} |tjjk(r|dkDr|jdd} | | fS|lj|j}tj ||| j|} tj||k7| d} | j} n"||k7jj|} |tjjk(r| j} | | fS|tj jk(r| j| z } | | fS)Nr"r#rr*r)rJr rrdrerPgatherrr!r'rxr9rrr7r)r()rwrrrrrrr wr safe_target_r rwsums r,_nll_loss_forwardrgsXXZFK z   A:E"(aE+  E"AAax++f 4fa@K((5Lll4l;CCKP PF [[</ ;FINN(((VaZ}}R- |##  HHTZZ ||A{L9AA+N{{6\14;xxz ,.33588> IMM''' <  inn** * , < r+c |jdkDr|jdksJd|jdksJd|jdk(xr|jdk(}|sA|jd|jdk(s"Jd|jd|jd|jd }|=|jdk(r|j|k(sJd |d |jt|||||S) Nrr#r+r"r,r-r.r/rNz/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rJr rr)rwrrrrr1 n_classess r,nll_loss_forwardr"s 88:>dhhjAoP/PP - ::<1 E 88:?8vzz|q'8L DJJqMV\\!_< $TZZL 6<<.J = 2I >fjjla/FLLNi4O 9)E++1<<. : P T669l KKr+c t|||||Sr4)r)rwrrrrs r,nll_loss2d_forwardr$s T669l KKr+Ac0|dz|z|dzz |z|zdzS)Nr#rr"r*r8r%s r,_upsample_cubic_convolution1r(s( UaK1q5 !Q & *Q ..r+c<||zd|zz |zd|zz|zd|zz S)Nrr4r*r's r,_upsample_cubic_convolution2r+s0 UQU]a !a% '1 ,q1u 44r+rcd}|jtjdk(rtj|d|z gd}tj|dzd|z gd}t||}t ||}tj |d\}}tj |d\}} ||| |fSt|dz|t ||t d|z |td|z |fS)Ngrrbrr|r)rrdrr+r(r) rr%tt1tt2w03w12rirlrjrks r, _upsample_get_cubic_coefficientsr1s Axx5<<&&kk1cAg,A.kk1s7C!G,!4*32*32cq)Bcq)B2r2~ )S! 4 (A . (q! 4 (q! 4   r+coeffstscPt|}tdt||DS)Nc3,K|] \}}||zywr4r*rrrs r,rz+_upsample_cubic_interp1d..sEHRREri)r1rr)r2r3coeffs2s r,_upsample_cubic_interp1dr8s$.r2G EFG0DE EEr+c6ttj|Sr4)rrdadd)r3s r,rrs %))R  r+ num_stepsc|dkrtjd||S|s|dz |z nd}tj| ||||S)Nr"rr)stepsrr)rdrdlinspace)r;rrrrs r,_linspace_from_neg_oner?sIA~||AfE::-:)a-9 $A >>1"ayu MMr+thetahrc|j}|j}t||||jd|d}t||||j|dd}t j d||}tj jj|ddd}tj jj|ddd}tj jj|d dd}||z|zS) Nr")r"r"r"r)rr#constantrrrorxr"r")r#r rrr?rrdrrrr) r@rArrrrgrid_xgrid_ygrid_ones r,_make_base_grid_4drJs KKE \\F$A}eV D I I!QPQ RF #A}eV D I I!QPQ RFzz)5@HXX $ $VjPQ $ RF XX $ $VjPQ $ RFxx""&&xV*TU&VH F?X %%r+rc|j}|j}t||||jdd|d}t||||jd|dd}t||||j|ddd} t j d||} tj jj|ddd}tj jj|ddd}tj jj| d dd} tj jj| d dd} ||z| z| zS) Nr")r"r"r"r"r)rrrCrrD)r"r#)r#r")rrrF) r@rrArrrrrGrHgrid_zrIs r,_make_base_grid_5drMs5 KKE \\F #A}eV D I I!QPQST UF #A}eV D I I!QPQST UF #A}eV D I I!QPQST UFzz,eFCHXX $ $VjPQ $ RF XX $ $VjPQ $ RF XX $ $VjPQ $ RFxx""&&xV*TU&VH F?V #h ..r+c|\}}}}t||||}|jddd|jjdzj d}|j|||dS)NrrNrr"rLr#)rJrrQrPr) r@rrrrQrAr base_gridgrids r,_affine_grid_generator_4drRsgJAq!Q"5!QmLI NN2q! $uxx'9'9!'< < A A" ED 99Q1a  r+c|\}}}}}t|||||}|jddd|jjdzj d} | j||||dS)NrOrNr4r"rLr)rMrrQrPr) r@rrrrQrrArrPrQs r,_affine_grid_generator_5drT#smMAq!Q"5!QOI NN2q! $uxx'9'9!'< < A A" ED 99Q1a ##r+ctjt|dvdt|dk(rt|||St |||S)Nrcy)NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r*r*r+r,r}z'affine_grid_generator..3rr+r4rO)rdr~rurRrT)r@rrs r,affine_grid_generatorrW-sJ LL D VU 4yA~(MRR(MRRr+rQinterpolation_mode padding_mode _expand_gridc  !"#$%&'()*+,-tjdvfdtjdvfddtdtdtffd -dtdtd tdtfd +dtdtdtf+fd dtdtdtf -fd }j\$%|j\})*}|d k(sJr(|j d)*|j )*d }dtdtdtf$%fd &tjjj dddtjjj ddddtdtdtdtf&)*fd dtdtdtffd "|d} |d} dk(r|| %} || $} | j| jc'('dz(}} '(dz}}| |}}|| z || z z}| |z || z z}| | z | |z z}| 'z | (z z}t"fd'(|f| ||f|||f|||ffDSdk(r<|| %} || $} | j}| j}"||dS-| %} -| $} | j'| j(| 'z ,| (z }s",jd,|jd}dtdtdtf "$%fd #dtdtf#'(,fd !t!fdtd D}t!||S)!N)rr"r#cdS)NzInvalid interpolation mode r*)rXsr,r}z"_grid_sampler_2d..Ls-.@-ABr+cdS)NzInvalid padding mode r*)rYsr,r}z"_grid_sampler_2d..Os-B<.+Qr+coordsrrKcBr|dzdz n|dz}|dzdz }||z|zSrr*)r^rrofsrs r, unnormalizez%_grid_sampler_2d..unnormalizeRs8%2tczCs Sj3|c!!r+ twice_low twice_highcP||k(rtj|S|dz }||z dz }||z j}tj||}||z j j tj }tj|dzdk(||z||z|z S)Nr#rr"r)rdrrfmodfloorr7int8re)r^rbrc coords_min coords_spancoords2extraflipss r,reflect_coordinatesz-_grid_sampler_2d..reflect_coordinates]s  "##F+ +] !I-2 J&++- 7K0;&--/222D{{ AINEJ. j0H50P  r+cdk(r|Sdk(rtj|d|dz Sr|dd|dz z}n|dd|zdz }tj|d|dz S)Nrr"r#rNr)r^rcoords_reflectedrrYrms r,compute_coordinatesz-_grid_sampler_2d..compute_coordinatesisx 1 M Q ;;vq$(3 3#6vq!tax.#Q #6vr1t8a<#P ;;/D1H= =r+c(||}||Sr4r*)r^r coords_unrpras r,compute_source_indexz._grid_sampler_2d..compute_source_indexus- "9d33r+r#r"rysc tjd|ktj|ktjd|k|kSrordr)rrtiHiWs r,in_bounds_condz(_grid_sampler_2d..in_bounds_condsF  GU&&rBw0A0A!r'2PR70ST  r+rwsc||rndt fd|jtj|jtj|fDS)Nr"c3pK|]-}tj|dj/ywr)rdrer)rrrrroHoWs r,rz1_grid_sampler_2d..clip..s7  KKa # ( (Ar2 6 s36r)rr7rdr) rrtrzrrrrrZryr}r~s @@r,clipz_grid_sampler_2d..clipsYb"% A1 ee%++e.EKK0H"M   r+ixiyc8 |||\}}}||f|zSr4r*) rrridx_xidx_yw_C_idxN_idxrrs r, get_summandz%_grid_sampler_2d..get_summands0B?ubue+,r11r+).r).r"rc3:K|]\}}}|||ywr4r*)rrrrrs r,rz#_grid_sampler_2d..s( R B " sc<|}|}||dSrWr*)rrr8rUrprrwrxs r,get_value_boundedz+_grid_sampler_2d..get_value_boundeds*#B+A#B+Aq!Q' 'r+r`c|dz z}dz ||dz|dz|f}t|S)Nr"r#)r8)r`iy_ofscsrix_nwiy_nwtxs r, get_coeffz#_grid_sampler_2d..get_coeffs[cAg&F!%!)V4!%0!%!)V4!%!)V4 B ,B3 3r+c3.K|] }|ywr4r*)rr`rs r,rz#_grid_sampler_2d..s:#y~:r4)rdr~rr~r rrrrrrfrrrPrrOr8).rrQrXrYrrZrsrQtwor8rUrrix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se ix_nearest iy_nearesttyr2rrrrrrprrrrwrxryrrr}r~rmrras.` ```` @@@@@@@@@@@@@@@@@@@r,_grid_sampler_2dr;s LLi'B LL !#Q"F"#"&"  F  s    PV   >F ># >& >4V4346477LAq"bZZNAr2s !8O8 yyAr2s+221aRC 6 v & LL188 , 1 1!Q1 =E LL188 , 1 1!Q1 =E    V    4F   22F2&22 V A V AQ !!R ( !!R (xxz288: uqy%ueaiueu urz*U urz* rEz*U rEz* t$t$t$t$      q !!R ( !!R (XXZ XXZ :z155 B  B    %Z %ZaBaB (& (f ( ( (  43 46 4 4:q::'33r+c"t|||||S)N)rQrXrYr)r)rrQrXrYrs r,grid_sampler_2drs   -!#  r+c&tjjdk(xrjdk(fdtjjdjdk(fdzj dS)Nr#r"cLdjdjS)Nzmatrix @ vector expected, got rr|rwrsr,r}zmv..s!0 BswwykJr+rcvdjddjddjddS)Nzsize mismatch, got input (rr8r"z), vec (r/rrsr,r}zmv..s<,TYYq\N!DIIaL>RURZRZ[\R]Q^^_`r+r|)rdr~rJrrrs``r,rrss LL  a*CGGINJ LL !  #` 3J    ""r+c|-|dz |zdz}d|z |z|tj|zz }nd|z |ztj|z }|||z}t||SrW)r logsigmoidr)rwrr pos_weightr log_weightrs r, binary_cross_entropy_with_logitsrsv  1n.2 F d"j1<<3E&EFF d"Q\\$%77 f} i 00r+tensor1tensor2is_outc  |j|jk\r||fn||f\}}ddlm |jdk\r|jdksy|jr|sy|jdk(ry |j dk(ry|j }|j }dg}t|ddD]}|j||dzt fd t|tt||DS) Nrrrr#FTr"rNc3JK|]\}}}|dk(xs||k(ywrkr*)rrrrrds r,rzshould_fold..-s4 D% TQY'845=8s #) rprfrdrrr rirrbrrrh) rrrt1t2t1_shape t1_strideexpected_striderrds @r, should_foldrs$+<<7<<#?gw gwEWFBJ GGqLRWW\ ||qBHHJ!O,xxH I cO!"&;tob&99:; !$ tH_56"  r+) pass_is_out)rc: |j}|j}|dk7r|dk7sJ|dk(r|dk(rtj||S|dk(r|dk(rtj||S|dk(rC|dk(r>tjtj tj |d|dS|dk(r|dk(rtj ||St|||r>||kD}|r |jn|}|s|n|dk(r|jn|}|j}t|dd} ttj| } |jdk(} | r| j|jd|j!| |d} | rWtj"j$j'| j || } |r| jj)S| Stj"j$j'| j|| S|dk\r^|dk\rX|dkDr|j+dnd}|j+d}|jdd}|dkDr|j+dn|j+d}|dkDr|j+dnd}g}t-|dz D]"}|j|j+|$|dk(rn|dk(ri|d|dk7r^|ddk(r'|j.rt1|j d|S|ddk(r'|j.rt1||j dSttj2||}|||gz}t5|}|j7|j!|||}|dk(}|r7||gz}|j7|j!||j d}n)|||gz}|j7|j!|||}|} |dkDr| j||dkDr| j||r/|j9|j dj;| S|j9|j;| Stj<ddy) Nrr"r#rNrLrFcy)Nz/both arguments to matmul need to be at least 1Dr*r*r+r,r}zmatmul..rr+)rJrddotrrrrPrrQrr rhrrrrbrr r _unsafe_viewrrrOrrPbroadcast_shapesrrbmmrr~)rrr dim_tensor1 dim_tensor2rwrrsizes_1 output_shape folded_dim1 t2_is_matrix t1_foldedrrm1 batch_tensor1m2rD batch_tensor2r\expand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded vector_rhstensor2_expand_sizetensor2_expandeds r,rPrP5s[++-K++-K !  q 00 0aK1,yy'**  kQ.xx))  kQ.}}UXXeoogq&A7KQOO  kQ.xx)) Wgv . +- $WZZ'$G+:J799;PW ((GCRL) X\\<8 vvx1}     ,JJ{GBK8 YY^^00b1A<PF-6699'') BF B99>>..y||B/?N N  kQ.!,aGLL Q \\"  cr* !,qW\\" gll26F +aGLL Q#% {Q' 2A  a 1 2 1 q a M!$44Q1$)>)>gooa0'::Q1$)>)>gwq'9:: $  " "=- @ 3aW<#$89#>>*=>FF !R !A% "6""= 23-r21  #7"a"@ &~~.ABJJ$b!  , ?    " ?    " #''(89AA"EJJ<X X#''(89>>|L L UUVr+rrc\j\}}t|d||}t|d||}tjtjj \}}t j|djj|} t j|djj|} t|| |} t|| |} | jd} | j} | j}| |z jdd}| | z jdd}| jt j} |jt j}|dz ||dz|d zf}| dz | | dz| d zft|t|}d \}j t j"k(rt%t%|}Dcgc]@}|dzzt j&|d zzjt j(Bc}|Dcgc]@}|d|zzt j&|d zzjt j(B}}fd fd t+fd|D}j t j"k(r|Jt-|||}nt/dt1||D}tj2}|j5|}|Scc}wcc}w)Nrr"rArrrNrrbr#r:rctj|ddz }tj|ddz }tjdd||g}|Sr)rdrrr0)rtry_idxx_idxrin_hin_wrs r, load_boundedz0upsample_bicubic2d_default..load_boundedsO B4!8, B4!8,   utT5%&@ Ar+ctfdD}jtjk(rJt |St dt |DS)Nc30K|] }|ywr4r*)rx_ofsrrUs r,rzCupsample_bicubic2d_default..get_x_interp..sBl1e,Bsc3,K|] \}}||zywr4r*r6s r,rzCupsample_bicubic2d_default..get_x_interp..sJRBGJri)rrrdrrrr)rUsrc_xrixs_ofsrweights_precision_x weights_xs` r, get_x_interpz0upsample_bicubic2d_default..get_x_interpsWB'BB ;;%++ %&2 22%eY8KL LJCy4IJJJr+c3.K|] }|ywr4r*)ry_ofsrs r,rz-upsample_bicubic2d_default..s;%,u%;rc3,K|] \}}||zywr4r*r6s r,rz-upsample_bicubic2d_default..sL(2rb2gLrirJ)r rr@rArCrrdrrr7rrPrfrrr1rrrrint16rrrrrr)rrrrrrQh_scale_factorw_scale_factorrr\rx_floaty_floatr8rUyscaleriys_ofs weights_yweights_precision_yrsrc_yr rKrrrrrrrs` @@@@@@@r,upsample_bicubic2d_defaultrs{{Aq$$D+a.-QN#D+a.-QN'' 5#H#H#U#UHAu  [^ELL9<<5<IA [^ELL9<<5<IA#NA}EG#NA}EG#G A Ak c *Fk c *F U[[A U[[A1uaQA&G1uaQA&G08I08I/9,, {{ekk!7 B7 B !** +ejjmc.A A E Eekk R  !** +ejjmc.A A E Eekk R   KK ;7; ;E {{ekk!"...#E96IJLc%6KLL//6M   ]  ;F MA  s9AL$AL)c $tjt|t|zdk(d|H|Jttt t ft dt |jdd|D}|r|nd\}}t|||||S)Nr"cy)Nz:Must specify exactly one of output_size and scale_factors.r*r*r+r,r}z(upsample_bicubic2d_vec..rr+c3PK|]\}}tt||z ywr4)rr)rrrjs r,rz)upsample_bicubic2d_vec..s*Au ! u,-s$&r#r:) rdr~rr rr~rr r)rrrrrrs r,upsample_bicubic2d_vecrs LL [D//14L((( #s(O  #AGGABK ?  )6}<GW %amWg VVr+c(fd}t||S)Nctj| ||zj}|dz |dz |jz jz S)Nrr")rdrrrrmiddlerdim_idxrs r,rz_reflection_pad..idx(sF,,ufunQXXFzVaZ'++-7<<>>>r+_reflection_or_replication_padrrrs` r,_reflection_padr"s ? *   r+c(fd}t||S)Nctj| ||zj}tj|d|dz S)Nrrr")rdrrrrs r,rz_replication_pad..idx9s6,,ufunQXXF{{7Avz22r+rrs` r,_replication_padr3s 3 *   r+idx_fncn t|dz tj|j dz dzfv fd|j d}|j z }t Dcgc]}|d dz |z z}}t Dcgc]}|d dz |z zdz}}|}t D]E}dg|jz} |||||||| ||z<t j|| }Gtj|} |j| }|Scc}wcc}w)Nr#r"c(dddzddzdS)Nreflection_padz d requires r"zD or r#zD inputr*r|sr,r}z0_reflection_or_replication_pad..Ls$.[q sQwiwOr+rJ) rurdr~rJr rOrr0r@rr) rrr inp_shapenc_dimr\ padding_left padding_rightr rrKrJs @r,rrDsH g,! C LL C!GS1W%%OI UUWs]F8=c C1GAq1-.CLC=B3ZHWQ#'A+.23HMH F 3Z1&**,. a)A, a@PQAJ##FC01 //7M   ]  ;F MDHs 1D-D2c t|dz|j dDcgc]}|dz  }}tDcgc]}|ddz |z z}}tDcgc]}|ddz |z zdz}}g}t|jD]c}dg|jz} d| |<|j t j |j||jj| e|d | d} dtDcgc] }| |||z} }tDcgc] }||| |z } }tDcgc]}d||z||z| |z } }tDcgc]}| |d||||z||zf}}tjtjtDcgc] }||c}}tj|| zd}fd}tjtDcgc]}gd c}D]}|t!dgzk(rg}g}tD]t}||dk(r | |}||}n=||dk(r| |}| |d||f}n$||dk(r| |}| |||||z ||dz f}|j |j v||||}|Scc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}w) Nr#r"rNrcF|\}}}tj||k\||kSr4rv) index_ranger\lbubs r,index_range_conditionz7_reflection_pad_backward..index_range_conditionus( 2r  b!r'22r+rrc .t D]*}||d||dk}t|ts%|s(|cStjt j |Dcgc] } | c}}t j ||zd}||zScc}w)Nr#r"r)rOr6rrGrrrr) rr index_rangesr\upper_less_than_lowerr rgrrJrhrs r,rz,_reflection_pad_backward..accumulatess A$0OA$6a9K$K !/6;P      CO PK "; / P   % %k4S# Fax QsB )rNrr")rur rOrprbrdrrrrGrrrr itertoolsrr)rhr8rrAdhwr\rrr view_shapexyzcenter left_reflect right_reflectrange_crrrrQareaoutsrrr rrJrs` @@@r,_reflection_pad_backwardr`s` g,! C''3$%. )Q1q5 )C )8=c C1GAq1-.CLC=B3ZHWQ#'A+.23HMHG 166]SS166\  1 u||AGGAJqxx@EEjQRS #A 3$%.C316c ;1c!f|A& ;F ;6;CjALOc!f,ALADI#JOqQQZ,q/1CF:OMO NSSVZHIAs1v Q/-2BBCG    eCjQ0<Q D  $ $[$F C HD !!c #C1J#CD4 5!s# #  s -AAw!|Qi%aj aB"1o"1vq,q/: aA#A&"1vs1v a0@'@#a&1*M KK     , -$l3-40 Kc *CH2<AOR&$Ds5 KK$$K)K.8K3K8K=L & Lrrrcjtj|||}tj|||}||fS)Nr)rdaminrE)rwrJrrrEs r,aminmaxr s2 ::dW 5D ::dW 5D :r+rctjtjtj|d||||S)Nrr)rrrdreisnan)rwrJrrs r,nansumr#s2 88EKK D 11d;S'QV8 WWr+rrrr/rc Ntjjd|d||||S)Nrr"r$rr start_step)r]rrrr/s r,arange_defaultr(s/ ;; ! ! 3vf " r+c Ntjj||d||||S)Nr"r$r&)r\r]rrrr/s r, arange_startr*s/ ;; ! ! sAU6&Z " r+cddlm}||i|S)Nr)out_dtype_dense)!torch._higher_order_ops.out_dtyper,)rBrCr,s r,out_dtype_decompr.sA D +F ++r+marginct tjtjjd jd tj|dk(xs|dk(dtjj dk(xr dk7fdtjj dk(xrj k( fdQtjtjj dk(xrj k( fdjdtjd}||z z}|jd}|dk(r|n||z}|z}tj j }tj|k7|d}|tjjk(r|j!S|tj"jk(r |j%|jdz S|j!d S) Nrr"r#cy)Nz only p == 1 and p == 2 supportedr*r*r+r,r}z#multi_margin_loss..rr+c"djSNzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: rrsr,r}z#multi_margin_loss..s_`e`k`k_lmr+c(ddjS)Nz#inconsistent target size, expected r r)nframersr,r}z#multi_margin_loss..s5fXYv||nUr+c(ddjS)Nz#inconsistent weight size, expected r r)rJrsr,r}z#multi_margin_loss.. s9#i ~Vr+rrr|)rd atleast_2d atleast_1dr r~rprrPrrRrrrer!r(rxrr)r) rrrDr/rrurfrrJr5s `` ` @@r,multi_margin_lossr:s   U #E   f %F [[^F ++a.C LLa!16#MN LL a$C1Hm LL q5V\\^v5U!!&) KK1  63!6 V   a F U0A UA AA !VQA  v  ,,s5<< 0C C6M1a(AINN(((vvx imm)) )uuw##vv!v}r+ is_targetc |j |jtj|}tj|}|jd}tjt dkxr|dk7 fdtjt dkxr k( fdtj ||j }|dk(}tjtj|||dd }||k}tj||d}tj|d| } tj||d} tj|| jd k(d } d | jjd z |z} | jd} | |z } tj| d| } |tjj k(r!| j#d j%} n@|tj&j k(r| j#} n| j#d } | j)|j*j-} | | fS)Nr"r#rcdSr3r*)orig_input_shapesr,r}z0multilabel_margin_loss_forward..,s_`p_qrr+cddS)Nzinconsistent target size: z for input of size: r*)r>orig_target_shapesr,r}z0multilabel_margin_loss_forward..0s,->,??STdSefr+rrNTrrr|rb)rrN)r rdr7r~rurrrreranyrPTrRr!r(rxrrr)r7rr)rrrrJris_endend_idx target_masktidx0r9tidx1r;rfr>r@s @@r,multilabel_margin_loss_forwardrHs{{    U #E   f %F ++a.C LL "/saxr LL !#M(9=M(Mf ,,s6== 1C r\FjjVS#6BMG-K KK VQ /E U%0A KK VR 0E #R!88a@I accmmm##e+A AA CA Iq!$AINN((( EEgE  # # % imm)) ) EEG EEgE  U[[)112CDI i<r+) attn_maskrjquerykey dropout_p is_causalrIc tjtjfdtjjdk(xr(jdk(xrjdk(fdtjdk(fdtjjdjdk(xrjdjdk(dt j j||d|\}}|jd d d djtj jd d d d}||fS) Nc"djS)Nz-query must be FP32, FP64, BF16, FP16 but got r)rJsr,r}z.is? }Mr+r4cndjdjdjS)Nz,q, k, v must be a 4 dimensional tensor, got rr|)rKrJrxsr,r}z.ms3>uyy{m2cggi[XZ[`[d[d[fZghr+rcdS)Nz&dropout probability must be zero, got r*)rLsr,r}z.ps$J9+"Vr+rcy)Nz&q, k, v should have the same head sizer*r*r+r,r}z.trr+)rIrLrM dropout_maskrjr#rr"rJ) rdr~r:rJr r"_scaled_dot_product_attention_mathr{rrrO) rJrKrxrLrMrIrjrattns ```` r,*scaled_dot_product_flash_attention_for_cpurV\s> LL &M LL q@SWWY!^@ q0@h LLSV LL A%++a.(KSYYq\U[[^-K8 ::BB   C LFDD q!Q" %"9"9 : Aq!  4<r+c.t|fd}|S)Nc<|i|}|dj|Sro)r)rBrCr outplace_ops r, inplace_opz$register_inplace..inplace_ops%4*6*Aw}}S!!r+r)aten_oprYrZs ` r,register_inplacer\s"G$"%" r+cB|js&|jst|}t|}tj||}t |t jr|dk7r||z}|dk(r|St |t jr|dk7r||z}||zSr )r:rr~rdrr6numbersNumber)rwbatch1batch2r^rir s r,baddbmmrbs  ! ! #DOO,=4yE  YYvv &F eW^^ , % qy dGNN +tqyd{ &=r+c2tj||dS)Nrfrnrp)rwrCs r, floor_dividerds 99T5 88r+c`tjtj|jdSrW)rGrrrr )rs r, sym_numelrfs   HLL!''1 55r+rrc|"tjj|g|Stjj|g||S)Nrrg)rr dim_IntList IntList_out)rwrrs r, sum_defaultrksC {xx##D"E#::xx##D"Es#CCr+c t|tjs|S|@tjj |t t|jStjj ||gSr4) r6rdrrrdimsrhrOrJ)rwrJs r,squeeze_defaultrns\ dELL )  {||  tE$((*,='>??||  u--r+c2tfdtt|jD}|jt j k(rt jnd}|jd|d|}|||j|jz z|fS)Nc3.K|] }|k7s |ywr4r*)rr\rJs r,rz)_weight_norm_interface..s@1qCxQ@s r#T)rr) rrOrur rrdrrrr7)rrrJkeep_dim norm_dtypers ` r,_weight_norm_interfacersst@c!''l 3@@H !5>> 9tJ 66!Xt:6 >D DGGAGG$$ %t ++r+ assume_uniqueinvertct|tjs!tj||j}t|tjs.|rtj ||Stj ||S|jdt|jdzkrt|||St||||S)Nrg$@g(\?rvrt) r6rdrrdrnerrrM isin_default isin_sorting)elements test_elementsrurvs r,isinr~s h -<<1E1EF mU\\ 2 88Hm4 488Hm4 4tc(..*:E&BBBHmFCC m=  r+)r-cB|?tj|jtj|j}n?tj|j|tj|j}||kj |j }|S)Nr)r-rr)rdrandrrrr7r)rwr-raw_prDs r, bernoullirsq  499;emmDKKP IIK--;;   $**%A Hr+rxcH|jdk(r%tj|tjS|jg|j d|j z}ttd|j dz d}||k(j|}|r|S|S)Nrr)r"rNr"r|) rrd empty_likerrr rprrOrA)r|r}rvr8rJr=s r,rzrzs~~1 ;; Dx~~D 0B0B)BDA b=---126 7C  " "s " +CC4"s"r+c|j}|j}|rtj||g}tj|d\}}|dd|ddk(} tj| ddgd} |r| j } tj | } | jd|| } | d|jStj|\} } tj| |} tj| | jk| d}| ||k(}|r|j n|}|j|jS)NT)stabler"rNrF) r*rdr!sortr` logical_notrrr searchsortedrerr )r|r}rurv elements_flattest_elements_flat all_elementssorted_elements sorted_orderduplicate_maskrsorted_test_elementsrQrtest_idxcmps r,r{r{s?$$&M&..0yy-1C!DE (- <(M%(,0DD..~1vuM +779N/q,?A())"'**-?"@a  !5}E;;s%9%?%?%AA3J"8, =#)coos{{8>>**r+c.|jd}||SrM)r)rwr flatteneds r,taker5s R I U r+c|tj}|tjk(r t|}tj ||j |Sr!)rdrOpreserve_formatrrresizer )rwrCrKs r, resize_asr<sD// ----e4 ;;tU[[ ; FFr+)F)noner4)r#)rNNr")rNFFrrrE)r"r"F)Fr)rrbN)rr"Nr:)NNN)rrFT)rrFr)rF(rGrr^rrjcollections.abcrenumrrrrrtypingr r r r r rdtorch._meta_registrations torch._primsr6rtorch._prims_common _prims_commonr@torch.nn.functionalrrrrrr torch._decomprr-rrrrrrtorch._prims_common.wrappersrrrr torch.utilsrr>torch.utils._pytreerr DispatchKeyrrhstr__annotations___opsr rr!rCrrIrDcompute_only_pw_cast_for_opmathpw_cast_for_opmathrpw_cast_for_int_to_realr~rRr[r]rgrrsrScalarryrrrrrrrrrrrrrrrrrrrrrr(rxr _safe_softmaxrrrr{rSrrrrrr)r2r5r;r>r@rBrFrZrbslicervrrarrrrrrrrr#r,r2r6py_implCompositeImplicitAutogradAutogradr5rHrLrRrZrrhrkrorzr|rrrrrrrrrrrrrrrrrr unsafe_chunkr rrno_statsrrrr%r)r+_fused_dropoutr.r;rrKrlift lift_freshr>rCrFrIrHr\rard_adaptive_avg_pool2drrrrrrrrrrrrr Generatorrrrrrr%r+r_upsample_nearest_exact1dr'r-rrr r r8rArErIrWr_rbrfrur~rnn_tanhrrrnn_relurrrrrrrrlstmrrrrgrurrrrrrrrrrrrrrrrr rr0rrrrr"r$r(r+r1r8rr?rJrMrRrTrWrrrrrrPupsample_bicubic2drrreflection_pad1dreflection_pad2dreflection_pad3drreplication_pad1dreplication_pad2dreplication_pad3drrreflection_pad1d_backwardreflection_pad2d_backwardreflection_pad3d_backwardrr r#rr3rr(r\r*r.r:rH+_scaled_dot_product_flash_attention_for_cpurVr\rbrdrfrrkrrJrnrsr~rrzr{rraddbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r hardswish_ hardtanh_hardtanh hardsigmoid___iand____and__ __ilshift__ __lshift__rr index_reduce_ index_reduce__ior____or__ __irshift__ __rshift____ixor____xor__ leaky_relu_ leaky_relulogit_logitrelu_rrenorm_renormround_rscatter_r scatter_add_ scatter_addscatter_reduce_scatter_reducesilu_r*r+r,rsM $%$77 #,,07 *(hh"" c zz~~   %!!99!!H#*88@@# uDDLL"uDDQQ   c f **+ \2F2v2,2--. \4v4&4/4../ \SS6SS5S0S ))* \           + 2))*+(,())*+"V","(() @f@@*@112 \fF3../ \PP%P05P@EP0P ' GFGvG(G//0 F&V1//0 \:F:&:U:1:001 \LL%L7<LNRL2L **+ \))f)3),)<**+5v5f5,5 " &v&&&#&**+ \>v>V>>,> **+66666,6334%% % % 66> %5%667                8 $112 \>3G3G1 1 1-01 1'1../ \11 &1061CF101**+0,0++, ^^)) 1 1 11  1-144<<=  % /5 BE MR > 44??@ R R  R  R R  R  RA R00889  % /5 BE NS : 00445 R R  R  R R  R  R6 R$$ $ $ V  $  $  $$ $@))* \NfNFNNN+N$../ \'' ' ' V  '  '  '' '0'T001 \   V       2B112  $^^)) 1 1 1 V 1 1  131&::; \ $^^))       V      <  --. ^^))1 1 11 1/1667 \ ^^))         8  " ))v)%)#),,- &&F&v&.&++,  O Oc O  O  O O  O- O ))* 6? 6? 6? C= 6? # 6?  6?+6?r  ( 4B  % ,4UO   5 & %kCCDk2236eHTN4E&++, VV A& AU A8D> A- A & S'*))* FT+,' $        (&556     7: DI  &\  &\ *T&\ tF| # " &\" ""J00$//2E2EFG !  &\     &    H   '')C)C)G)GH "&   c    $v,   d6l   ))00155C5c5%PS BT52544<<=67B B $S B03B 63;B>B  ))*//C/c/% :L/+/"--55)) / / &/ / 63; //@ # fF##$"../            0" #   f 6   $ 77??@S&S& S& S&  S& F  S&  S& S& S& S&dS& 8F Xf-x/? ?@S&AS&n77;;<     F       d ,, ,, ,, 8F Xf-x/? ?@=8 8F# x/? 77??@NN N3iN  N  N V  N 6 NdN 8F Xf-x/? ?@NANd77;;< 3i     V   6 d ,, ,, ,, 8F Xf-x/? ?@=4O O V O 6 O6" O &! O  OO OO 6668F#3Xf5E EFOd../ UK/ (  ( V  ( 6  (6" ( &! (  ( (  ( 666 !" (00 (4'' (<(<='' (M(MN    V    6   6"  &!          666 !"  O>  F"";#H#HIG4<GJGAAIIJ  V  6        666 !"K*55==> (  ( V  ( 6  ( (  (  ( (  ( 666 !" (? ( 55>>? (  ( V  ( 6  ( (  (  ( 666 !" (@ (@@HHIK K V K 6 K K  K  KK K 66666 12KJK0  V  6       :44<<=1 1 V 1 6 1 1  1  1 1 6666 )*1>14??GGHC C V C 6 C C  C  C C 666666 9:CIC422::;1 1 V 1 6 1 1  1  1 1 6666 )*1<14++, VV- & $( %)37& VZ  & EKK & U\\ " &  &&E//0&'&XdiiAB C &&{';';<--. VVVV,   6 6"  &!   !&-/=> 00889  V 6"  &!   &!  d 68F#Xf%5 56:677??@dd d V d6" d &! d  d&!d d ddd 68F#Xf%5 56dAdP77;;<"" " V "6" " &! "  "&!" " "d" ,," ,," ,," 68F#Xf%5 56"="J778 VVV$  6"  &!   v%90667 VVV$  6"  &!   v%82112 c'vc'E#s(Oc'3c'L )8(!2( % $'$'+/    tS%   c5 ! ( &   &;'; 8 //334//334//334$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9: 3  3$s)$ 3DK( 3 3;L;L;L555 366::;66::;66::;##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A ?  ?$s)$ ?DK( ? ?BSBSBS<<< ?@0088$:Q:Q:U:UVW  (()N)NO  (()=)=> Dd;#; ;c; UO; ;<?PX; ##++T-K-K-O-OP''// 0U0UV''// 0D0DE Dd;#G GcG UOG G<FW G0088$:Q:Q:U:UVW  (()N)NO  (()=)=> Dd;!% $ G GcGuoGuo G  G<?PXG ##++T-K-K-O-OP''// 0U0UV''// 0D0DE Dd;!% $ S ScSuoSuo S  S<FW S0088$:Q:Q:U:UVW  (()N)NO  (()=)=> Dd;!% $ $ Q QcQuoQuo Q uo Q  Q<?PXQ ##++T-K-K-O-OP''// 0U0UV''// 0D0DE Dd;!% $ $   c uo uo  uo   <FW    c % !    4  >/ FK+\&,/-d& R ++,[BBC[112.3D-.8 ++,[BBC[112.3D-.8 **+KAABK001.2C,.@ **+KAABK001.2C,.@ /6=@/d (>>?--.S/@)S> ' ==> ,,-S.?(S<;; &{<<={++,.->'.6' ==> ,,-..?(.644889!!))+*O*OP!!))+*>*>?@Q:33778  (()N)NO  (()=)=>?P90044511556##K$I$IJ##K$8$89%%k&K&KL%%k&:&:;&&{'L'LM&&{';';<A=N<M:K76A //779O9O9S9STU  !% K KcKKuo K  KVK %%t'?'?'C'CD!!))+*>*>?  !% $ U UcUUuo U uo U  U@ U &&(A(A(E(EF  !% $ $   c  uo  uo  uo    V/7 & 7$,V$47IO7 7-'9-f-E EcEE % ! E  EER))112Fv$3,,d.?.?@A B++,-"."//019292234C5C6AABCMDM22 2 2  V 2  2  2  66> 2 j--. X~&L L L V L L  L  66> L'/L6//0 X~&L L L V L L  L  66> L'1L/F/u//5F5u55  3E (F%7FVFF !Xf%!&!NN#'N05 NEJ\\N&f&&&T&"/f////T/"!V!49!T!$V$49$T$223 SStCySS4S  c4 c4 c4c4 c4  c4  c4 c4L,,-             .     #! #==> $)..:N:N 1? 1 dtB[BBC>>? ',tW@DtWn0088$:Q:Q:U:UVW  (()=)=>  $# P PsCxPPe_ P e_ P  P?XPf//334$$[%J%JK$$[%9%9:  48 W W%S/*WWE%,/0 W  W;L5 W.--.--.--.  v c3h F /// ../../../   sCx V 000   38_ c3_f, - 8667667667 \T888Tn % UEu&  $ X4X%X,,dkkoo>? $( ==%)    EKK   LL  U\\ "   @ **+, $( ==%)      EKK  LL  U\\ "  -  ",#, ../'' (<(<= #^^)) ) ) ))  ) V  )  ) )>0)X;;<$$,,44[5I5IJ X{#- - -- 66> -$K=-xHHPPQ  @#'!@ @ @ @ @  @@ E?@ 66>@R@F'  ( ))* 9+9'6(6))488<<89$( D  D EKK  D &  D  D: D--t||/?/?@A.&.x}.B. ==>,?, " 38 # $../,0 ,, (  \\  0 $5:#