2VhddlZddlZddlZddlmZddlZddlZddlm Z ddlm Z ddlm Z ddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZGddej<ZddZ GddeZ!y)N)partial)backend) callbacks) optimizers)tree)config)distribution_lib)trainer) array_slicing)data_adapter_utils) EpochIterator)traceback_utilscneZdZfdZ ddZdZdZdZdZddZ ddZ dd Z dd Z e j dd Ze j dd Ze j dd Z ddZ ddZdZdZdZdZ ddZ ddZ d dZxZS)! JAXTrainercZt|d|_d|_d|_d|_y)NT)super__init__train_function test_functionpredict_function_jax_state_synced)self __class__s M/home/dcms/DCMS/lib/python3.12/site-packages/keras/src/backend/jax/trainer.pyrzJAXTrainer.__init__s. "! $!%c .i} |jr|| d<|j|||fddi| \} }} | r!|jj| |_|j |||||| ||\} } | r|jj| \}}}| }|ro|j ct t|j j|}tj|5|j j| } ddd| || ||ffS#1swYxYw)z?This method is stateless and is intended for use with jax.grad.training return_lossesT)xyy_pred sample_weightrN state_mapping) _call_has_training_argstateless_call_losses_overrideclearstateless_compute_loss optimizerlistzip variablesrStatelessScope scale_loss)rtrainable_variablesnon_trainable_variablesmetrics_variablesrr r"roptimizer_variableskwargsr!losseslossr- unscaled_lossmappings rcompute_loss_and_updatesz#JAXTrainer.compute_loss_and_updatessS  & &!)F: 3F$2E2E  # 3  3  3 /'   ! ! ' ' )$*D !55  # '6  i   ! ! ' ' )  J 57H  23t~~779LMNG''g> 7~~006 7   #      7 7s D  Dctjt|j|Dcgc] \}}||f c}}5} |jj |t j|djd|j||||} dddg} |jD]2} j|} | |j} | j| 4 | fScc}}w#1swYVxYw)Nr#r)r") rr.r,r2 _loss_tracker update_staterflattenshapecompute_metricsget_current_valuevalueappend) rr2r7rr r!r"ref_vvscopelogsnew_metrics_variablesnew_vs r_update_metrics_variablesz$JAXTrainer._update_metrics_variablesZs # #!$D$:$:4+>+> !8!, ?E, ((  , , $$; $" -  # ///rcjdkDrC|r9djs!jrtjfd}|Sfd}|Sfd}|S)NcT|d}|ddD]}tjd||}|S)Nrr_cDtjj||gSN)rMnumpy concatenate)t1t2s rz@JAXTrainer._make_function..concatenate..s399+@+@"b+Jr)r map_structure)r\output next_outputs rrdz.JAXTrainer._make_function..concatenates@$QZF'.qr{ !%!3!3J"'" "Mrct|}||\}}|g} tjdz D]*}t|}||\}}|j|, |}||fS#t$rYwxYwNr_)nextrangesteps_per_executionrB StopIteration) rQiteratorrRr\rY_outputsrdr step_functions r iterator_stepz0JAXTrainer._make_function..iterator_steps>D%25$%?NGU&iG!&t'?'?!'C!D5A#'>D.;E4.HOHe#NN845 *'2G"E>))sAA// A;:A;ct|}||\}} tjdz D]}t|}||\}} ||fS#t$rY||fSwxYwrl)rmrnrorp)rQrqrRr\rYrrss rrtz0JAXTrainer._make_function..iterator_steps>D%25$%?NGU!&t'?'?!'C!DHA#'>D-:5$-GNGUH #E>))"E>)s4A A"!A"c(|t|Srb)rm)rQrqrss rrtz0JAXTrainer._make_function..iterator_step s$UDN;;r)ro run_eagerly jit_compilerMjit)rrsconcatenate_outputsrtrds`` @r_make_functionzJAXTrainer._make_functionsb  # #a '""''D,<,<"%''+"6K *@! *  <rc|j|sy|js.|jr"tj|j d}n |j }|j |}||_yNrdonate_argnums)rrwrxrMryrVr{)rforcerVrss rmake_train_functionzJAXTrainer.make_train_functions[    *5 D$4$4 CJJ++J7 +rc|j|sy|js.|jr"tj|j d}n |j }|j |}||_yr})rrwrxrMryrZr{)rrrZrss rmake_test_functionzJAXTrainer.make_test_function s[    )% D$4$4 qAII++I6 *rcj|s jSfd}js#jrtj|d}j |dfd}|_y)Nc@j||\}}||d|ffS)Nr)r])rQrRr\r1rs rr]z6JAXTrainer.make_predict_function..predict_step4s0/3/@/@/M ,G,U1X'>?? ?rrr~T)rzc$||\}}||fSrb)rQrqr\_step_functions rrsz7JAXTrainer.make_predict_function..step_function?s+E8 !r)rrwrxrMryr{)rrr]rsrs` @rmake_predict_functionz JAXTrainer.make_predict_function0so  ,U(( ( @D$4$477<BL,, d-  "!.rc |jdtj}|r||krtjd|z|}d|_|r#|!t j||| f|\\}}} }|tj|\}}}t||| || | | |j}|j||jt|tj s)tj |d|dk7|||j"|}|j%|j'd |_i}d }|j+|j,xs| } t/| |D]}|j1|j3|d|_|j75|D]\}}|j9||j4r|j;ddddd }d |_|j=|\}}|\}}}} |||| d |_|jA|||j(snddd|jCtE|jG}!||jI||rtK|d d%t|xs||j|d |_|jM|xs|||dd}"|"jOD#$cic] \}#}$d|#z|$ }"}#}$|!jQ|"|jS||!|!}|j(snd}|jCt|jTtVjXr*|dkDr%|jTj[|j\tK|d d|`|r|j_|d|_|ja|jbS#1swYxYwcc}$}#w#|jCt|jTtVjXr*|dkDr%|jTj[|j\tK|d d|`|r|j_|d|_|jawxYw)NfitzLimiting epochs to %d)validation_split)rr r" batch_sizesteps_per_epochshuffle class_weightrorqTr) add_history add_progbarverboseepochsstepsmodelFr0r1r3r2purge_model_variablesrX_eval_epoch_iterator)rr r"rrorr)rr r"rrr return_dict_use_cached_eval_datasetval_)rF)2_assert_compile_calledr max_epochswarningswarnrr train_validation_splitr rLJAXEpochIteratorro_symbolic_buildreset isinstancecallbacks_module CallbackList num_batches$_record_training_state_sharding_specr stop_trainingon_train_begin_initial_epochrn reset_metricson_epoch_beginrcatch_stop_iterationon_train_batch_begin_get_jax_stater _jax_stateon_train_batch_endjax_state_syncdict_get_metrics_result_or_logs _should_evalgetattrevaluateitemsupdate on_epoch_endr*optimizers_module Optimizerfinalize_variable_valuestrainable_weights on_train_end_clear_jax_state_shardinghistory)%rrr rrrrrvalidation_datarrr" initial_epochrvalidation_stepsvalidation_batch_sizevalidation_freqrval_xval_yval_sample_weightepoch_iterator training_logstraining_finishedepochsteprqrQrFr0r1r3r2 epoch_logsval_logsnamevals% rrzJAXTrainer.fitEs( ##E*&&( *v- MM1J> ?F$(!  7 44A}%8H %A}  & #;;OL  !*'!+% $ 8 8   n5)%5%B%BC(55 #qL$00I 113   "" !  "++<} l -}f5Y ""$((/)-&#88:'"*8&"h!66t< 11$($7$7488<48266: %8%E6;D2&*&9&9%&J e " /3/- 4G7N3F1B +"44T4@--"M&"'"Z##%"$"B"B4"HI #.43D3D?4t%;TBJ4D##*;'<'J 040H0H,<$)51 $}}&7#8#FJ."+$(15 - H=ENN>+<+F+FGQJ778N8NOt3T:F- &&M&:"DO  * * ,||O'"'"X     !4>>+<+F+FGQJ778N8NOt3T:F- &&M&:"DO  * * ,s@AO$B O.O1B(OO)4OOO OBQ5c B|jd| jdd} | rtd| | r |j} nt |||||d|j } |j | | jt|tjs(tj||dk7|d| j| }|j|jd|_|ji} |j!d |_| j%5| D]\} }|j'| |j"r|j)d d d d }d|_|j+|\} }|\}}}|||d |_|j/| | |jsnddd|j1|j3| } |j5| d|_| s|j7|r| S|j9| S#1swYixYw) NrrFzArguments not recognized: )rr r"rrrrorrr_rrrrrTr0r1r2rr0r1r2)rpop ValueErrorrrrorrrrrrrrstop_evaluating on_test_beginrrron_test_batch_beginrrron_test_batch_endrr on_test_endr_flatten_metrics_in_order)rrr rrr"rrrr4use_cached_eval_datasetrrFrrqrQr0r1r2s rrzJAXTrainer.evaluatesC ##J/"(**-G"O 9&BC C "!66N.+% %$($<$<N n5)%5%B%BC(55#qL$00 I 113 !$! !%  0 0 2# "0" h--d3)) //,004*..2 0E .3D*"00A e  '+%,?/F): #++D$7''E" # L //5d#&  * * , K--d33a# # s#BH*HHct|||d|j}td|jDsb|D]M\}}t j t |\}}}tj5||dddn|jt|tjs(tj||dk7|d|j|}|j|jd|_|j#d} d|_d} d} |j'5|D]\} }|j)| |j$r|j+ddd } d|_|j- |\}} | \}} || d |_| || } |j1| d |i|j snddd|j3|j5d|_|j7t9j:t<j>| S#1swYxYw#1swYrxYw) NF)rrrrroc34K|]}|jywrbbuilt.0layers r z%JAXTrainer.predict..zC55;;Crr_rcn|tjd|}|Stj|d|||S)Nc|gSrbr) batch_outputs rrgz?JAXTrainer.predict..append_to_outputs..s,rc$|j|Srb)rB)rirs rrgz?JAXTrainer.predict..append_to_outputs..s|1Lr)rrhmap_structure_up_to) batch_outputsr\s rappend_to_outputsz-JAXTrainer.predict..append_to_outputssG,,7!N ((!L!  NrT)r0r1rr0r1r\) rroall_flatten_layersr rLrmrr.rrrrrrrstop_predictingon_predict_beginrron_predict_batch_beginrrron_predict_batch_endron_predict_endrrrnprd)rrrrrrrrYrqrr\r1rrQrr0s rpredictzJAXTrainer.predictmsS *!! $ 8 8  CD,@,@,BCC-  8,EEN1a++-G   ")%5%B%BC(55#qL$00 I 113 ""$$""$ "&"&  0 0 2 "0 h006)) //,004.20E .3D*'+'<'.data"Aq-#89 9 data_batchTFrrXc,tj|Srbrarrayrs rrgz+JAXTrainer.train_on_batch..BHHQKr)rrr class_weight_to_sample_weightsrrmrrrrrrrrrhr) rrr r"rrrRrQrFr0r1r3r2s ``` rrzJAXTrainer.train_on_batchsD ##$45  #( //$$0>3 /MM<M : TV 5 113   "## $$( $""' $ "'))%8 e    #  $7'>#6!2   !!"7> K--d33rc|jdfd}|jt||j|j |j dddd}d|_|j||\}}|\}} } || | d|_|jtjd|}|r|S|j|S) N test_on_batchc3.KtfywrbrrsrrRz&JAXTrainer.test_on_batch..datarrrTFrrc,tj|Srbr r s rrgz*JAXTrainer.test_on_batch..-r r) rrrmrrrrrrrrrhr) rrr r"rrRrQrFr0r1r2s ``` rrzJAXTrainer.test_on_batchs ##O4 : TV 5 113 !## $$(""' $  "'((7 eKPG46G#6'>!2  !!"7> K--d33rctd|jDs%tj5|ddd|j |j |j dddd}d|_fd}|j||\}}|\}}||d|_ |jtjd|}|S#1swYxYw)Nc34K|]}|jywrbrrs rrz.JAXTrainer.predict_on_batch..3rrTFrc3Kfywrbrr srrRz)JAXTrainer.predict_on_batch..dataBs$Js rc,tj|Srbr r s rrgz-JAXTrainer.predict_on_batch..LsRXXa[r) rrrr.rrrrrrrrrh)rrrQrRrr0r1s ` rpredict_on_batchzJAXTrainer.predict_on_batch2sCD,@,@,BCC'') Q  113 ""$## $$(#"' $  "'  $44UDFC u7<44#6'>  **+@-P 1  s CC#ct|ddr |jry|jjdd}|jjdd}|jjdd}|jjdd}|r/t |j |D]\}}|j ||r/t |j|D]\}}|j ||r9t |jj|D]\}}|j ||r/t |j|D]\}}|j |d|_y)Nrr0r1r3r2T) rrrgetr,r0assignr1r*r-r2)rr0r1r3r2rCrDs rrzJAXTrainer.jax_state_syncOsLt\40D4J4J "oo112GN"&//"5"5 %t# #oo112GN OO//0CTJ  8 8:MN q Q ",,.E q Q   8 8:MN q Q  6 68IJ q Q !%rc|jDcgc]}|jjc}|_|jDcgc]}|jjc}|_t |drJ|j>|jjDcgc]}|jjc}|_ ng|_ |jDcgc]}|jjc}|_ ycc}wcc}wcc}wcc}w)Nr*) r0rAsharding_trainable_variable_shardingsr1!_non_trainable_variable_shardingshasattrr*r-_optimizer_variable_shardingsr2_metrics_variable_shardings)rrDs rrz/JAXTrainer._record_training_state_sharding_specis&*&>&>. !"AGG  . *'+&B&B2 !"AGG  2 . 4 %$..*D*...*B*B2%&  2D .24D .&*&<&<, !"AGG  , (. 2 2 , sC8C=DDc<d|_d|_d|_d|_yrb)rrrr rs rrz$JAXTrainer._clear_jax_state_shardingzs"-1*15.-1*+/(rc|xsg}|xsg}|xsg}|xsg}tt|D]5}tjj |||j |||<7tt|D]5}tjj |||j |||<7tt|D]5}tjj |||j|||<7tt|D]5}tjj |||j|||<7||||fS)aEnforce the sharding spec constraint for all the training state. Since the output of the train/eval step will be used as inputs to next step, we need to ensure that they have the same sharding spec, so that jax.jit won't have to recompile the train/eval function. Note that this function will also rely on the recorded sharding spec for each of states. This function is expected to be called within the jitted train/eval function, especially around the end of the function. ) rnlenrMlaxwith_sharding_constraintrrrr )rr0r1r3r2is rrPz&JAXTrainer._enforce_jax_state_shardingss&27R"9"?R17R-3s./0 A%(WW%E%E#A&(J(J1(M&  " s234 A),)I)I'*66q9* #A &  s./0 A%(WW%E%E#A&(J(J1(M&  " s,-. A#&77#C#C!!$d&F&Fq&I$ a   #     rc|r|jD] }d|_ |r|jD] }d|_ |r"|jjD] }d|_ |r|j D] }d|_ yy)aRemove all the model variable for memory saving. During JAX training, since the training function is stateless, we have to pass in and get the model weights over and over, during which the copy of the weights that attached to the Variable are still and occupying extra memory. We remove those variable to save memory (for better memory utilization) at the beginning of the epoch, and reattach the value back to variables at the end of the epoch, via `jax_state_sync()`. N)r0_valuer1r*r-r2)rr0r1r3r2rDs r_purge_model_variablesz!JAXTrainer._purge_model_variabless" --  "11  ^^--  ++   rc.g}|r3|j|jDcgc]}|jc}|r3|j|jDcgc]}|jc}|r=|j|jj Dcgc]}|jc}|r3|j|j Dcgc]}|jc}|r|j||||t|Scc}wcc}wcc}wcc}w)NrX) rBr0rAr1r*r-r2r*tuple)rr0r1r3r2rrQrDs rrzJAXTrainer._get_jax_states  LL4+C+CDa!''D E " LL4+G+GHa!''H I  LL4>>+C+CDa!''D E  LL4+A+ABa!''B C  ' '$7(?$7"3 (  U|EHDBsDDD D)FN)F)NNNr_autoNgNTNNrNNNr_)NNNr-NNNF)Nr-NN)NNNF)NNF)NNNN)FFFF)FFFFF)__name__ __module__ __qualname__rr9rIrVrZr]r{rrrrfilter_tracebackrrrrrrrrrrPr*r __classcell__)rs@rrrs`& 9 v+,*X(T0.1f, + .*%%  "#{&{z%%  g4&g4R%%HL[P&[P@  ;4@  (4T:&4 "0! $  . d" %!  @" %!# rrc&tjW|tjfd|}t t j j}tj|||Stjtj|S)Nc:j|jSrb)get_data_layoutr>d distributions rrgz"_distribute_data..s,66qww?r)batch_dim_name) r r8rrhrjax_distribution_libdistribute_data_inputr9rM device_put)rRlayoutsjax_dist_data_inputr8s @rrrs#002L ?((?G& 6 6'66 !!"5tWEE   cnnd 33rc$eZdZdZdZdZdZy)rc,t|jSrb)rm_epoch_iteratorr"s r__next__zJAXEpochIterator.__next__sD(())rctj}||j|S|j|jj Srb)r r8_get_distributed_iterator_prefetch_numpy_iterator data_adapterget_jax_iterator)rr8s r _get_iteratorzJAXEpochIterator._get_iteratorsL'446  #11,? ?,,    . . 0  rc#Kd}|jjD]+}|tjfd|}t ||-yw)zALazily compute layouts to reduce host to device transfer latency.NcNj|jjSrb)r5r>backend_layoutr6s rrgz.s!l::$nr)rFrGrrhr)rr8r=rRs ` rrDz*JAXEpochIterator._get_distributed_iteratorsX%%668 2D,,%  #41 1 2sA Ac#Ktjdfd }|drj|dryyw)aShard and prefetch batches on device. Most of the implementation has been borrowed from `flax.jax_utils.prefetch_to_device` This utility takes an iterator and returns a new iterator which fills an on device prefetch buffer. Eager prefetching can improve the performance of training loops significantly by overlapping compute and data transfer. cptj|D]}jt|yrb) itertoolsislicerBr)nrRnumpy_iteratorqueues renqueuez:JAXEpochIterator._prefetch_numpy_iterator..enqueue%s1!((; 5 -d34 5r)rQr_N)rM) collectionsdequepopleft)rrRrTrSs ` @rrEz)JAXEpochIterator._prefetch_numpy_iterators@!!#  5 ! --/ ! AJs AA A N)r.r/r0rBrHrDrErrrrrs*  2rrrb)"rUrOr functoolsrrMrcr keras.srcrrrrrrkeras.src.backendrr r:keras.src.distributionkeras.src.trainersr base_trainer keras.src.trainers.data_adaptersr r !keras.src.trainers.epoch_iteratorr keras.src.utilsrTrainerrrrrrrrbsa 35$F36:?;+O%%Od4$1}1r