Ë
    þÍ:jI6  ã                  óÄ  — d dl mZ d dlZd dlmZ d dlZd dlmZ d dl	m
Z
 erd dlmZ d dlZd dlmZ d dlmZ nd dlmZ  ed	«      Z ed
«      Z ed«      Z e
e«      Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdddœ	 	 	 	 	 	 	 	 	 dd„Zddddddœ	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zy) é    )ÚannotationsN)ÚTYPE_CHECKING)Ú*single_blas_thread_if_scipy_v1_15_or_newer)Ú
get_logger)Úbatched_lbfgsb)ÚBaseAcquisitionFunc)Ú_LazyImportzscipy.optimizeÚtorchzoptuna._gp.batched_lbfgsbc                óš  ‡ ‡‡— |j                   dk(  sJ ‚t        ‰«      dk(  r'||t        j                  t        |«      t        ¬«      fS 	 	 	 	 	 	 d	ˆ ˆˆfd„}t        «       5  t        j                  ||dd…‰f   ‰z  |j                  «       D �cg c]  }|‘Œ c}f‰D �cg c]	  }dd|z  f‘Œ c}t        j                  |«      d¬«      \  }	}
}ddd«       |j                  «       }	‰z  |dd…‰f<   
 }||kD  dkD  z  }t        j                  |dd…df   ||«      t        j                  |||«      |fS c c}w c c}w # 1 sw Y   ŒsxY w)
a'  
    This function optimizes the acquisition function using preconditioning.
    Preconditioning equalizes the variances caused by each parameter and
    speeds up the convergence.

    In Optuna, acquisition functions use Matern 5/2 kernel, which is a function of `x / l`
    where `x` is `normalized_params` and `l` is the corresponding lengthscales.
    Then acquisition functions are a function of `x / l`, i.e. `f(x / l)`.
    As `l` has different values for each param, it makes the function ill-conditioned.
    By transforming `x / l` to `zl / l = z`, the function becomes `f(z)` and has
    equal variances w.r.t. `z`.
    So optimization w.r.t. `z` instead of `x` is the preconditioning here and
    speeds up the convergence.
    As the domain of `x` is [0, 1], that of `z` becomes [0, 1/l].
    é   r   ©Údtypec                óü  •— t        j                  |«      }| j                  dk(  r|j                  dk(  sJ ‚| ‰	z  |d d …‰f<   t        j                  |«      j                  d«      }‰j                  |«       }|j                  «       j                  «        |j                  j                  «       j                  «       }t        j                  |j                  «       j                  «       «      }||d d …‰f   ‰	z  fS )Nr   T)ÚnpÚarrayÚndimr
   Ú
from_numpyÚrequires_grad_Ú	eval_acqfÚsumÚbackwardÚgradÚdetachÚnumpyÚ
atleast_1d)
Úscaled_xÚfixed_paramsÚnext_paramsÚx_tensorÚ	neg_fvalsÚgradsÚ
neg_fvals_ÚacqfÚcontinuous_indicesÚlengthscaless
          €€€úk/home/mcse/projects/srt_converter/srt-converter-venv/lib/python3.12/site-packages/optuna/_gp/optim_mixed.pyÚnegative_acqf_with_gradz9_gradient_ascent_batched.<locals>.negative_acqf_with_grad8   sÝ   ø€ ô —h‘h˜|Ó,ˆà�}‰} Ò! k×&6Ñ&6¸!Ò&;Ð;Ð;Ø-5¸Ñ-Dˆ’AÐ)Ð)Ñ*ä×#Ñ# KÓ0×?Ñ?ÀÓEˆØ—^‘^ HÓ-Ð-ˆ	Ø�‰‹× Ñ Ô"Ø—‘×$Ñ$Ó&×,Ñ,Ó.ˆÜ—]‘] 9×#3Ñ#3Ó#5×#;Ñ#;Ó#=Ó>ˆ
ð ˜5¢Ð$6Ð!6Ñ7¸,ÑFÐFÐFó    Né   éÈ   )Úfunc_and_gradÚ
x0_batchedÚbatched_argsÚboundsÚpgtolÚ	max_iters)r   ú
np.ndarrayr   zlist[np.ndarray]Úreturnútuple[np.ndarray, np.ndarray])r   Úlenr   ÚzerosÚboolr   r   ÚcopyÚmathÚsqrtÚwhere)r#   Úinitial_params_batchedÚinitial_fvalsr$   r%   Útolr'   ÚparamÚsÚscaled_cont_xs_optÚneg_fvals_optÚn_iterationsÚxs_optÚ	fvals_optÚis_updated_batchs   `  ``          r&   Ú_gradient_ascent_batchedrF      sv  ú€ ð. "×&Ñ&¨!Ò+Ð+Ð+Ü
ÐÓ !Ò#Ø% }´b·h±h¼sÀ=Ó?QÔY]Ô6^Ð^Ð^ðGØðGØ,<ðGà	&÷Gô" 
4Ó	5ñ 
Ü:H×:WÑ:WØ1Ø-ªaÐ1CÐ.CÑDÀ|ÑSØ.D×.IÑ.IÓ.KÖL Uš5ÒLÐNØ(4Ö5 1�Q˜˜A™’JÒ5Ü—)‘)˜C“.Øô;
Ñ7Ð˜M¨<÷
ð $×(Ñ(Ó*€FØ$6¸Ñ$E€FŠ1Ð Ð Ñ!ð �€IØ! MÑ1°lÀQÑ6FÑGÐô 	�‰Ð!¢! T 'Ñ*¨FÐ4JÓKÜ
�‰Ð! 9¨mÓ<Øðð ùò MùÚ5÷
ð 
ús*   Á".EÂ	D7ÂEÂ D<Â.!EÄ7
EÅE
c                ó$  — t        |«      dk(  r||dfS ||||   k7     }t        j                  |d d d …f   t        |«      d¬«      }||d d …|f<   | j                  |«      }t        j                  |«      }||   |kD  r||d d …f   ||   dfS ||dfS )Nr)   Fr   )ÚaxisT)r4   r   ÚrepeatÚeval_acqf_no_gradÚargmax)	r#   Úinitial_paramsÚinitial_fvalÚ	param_idxÚchoicesÚchoices_except_currentÚ
all_paramsÚfvalsÚbest_idxs	            r&   Ú_exhaustive_searchrT   a   s³   € ô ˆ7ƒ|�qÒà˜|¨UÐ2Ð2à$ W°¸yÑ0IÑ%IÑJÐä—‘˜>¨$²¨'Ñ2´CÐ8NÓ4OÐVWÔX€JØ5€JŠq�)ˆ|ÑØ×"Ñ" :Ó.€EÜ�y‰y˜Ó€HàˆX�˜Ò%Ø˜(¢A˜+Ñ&¨¨h©¸Ð=Ð=à˜<¨Ð.Ð.r(   c                ó¦  ‡ ‡‡‡‡‡— t        ‰«      dk(  r||dfS dˆfd„} ||‰   «      }t        j                  |‰   ‰|   «      sJ ‚|| iŠ|j                  «       Šdˆ ˆˆˆˆfd„Šdˆˆfd„}d}	t	        j
                  |‰d   |	z
  ‰|   ‰d   |	z   fd	|¬
«      }
 ||
j                  «      } ‰|«       }||k7  r||kD  r‰|   ‰‰<   ‰|dfS ||dfS )Nr)   Fc           	     óÞ   •— t        t        j                  t        j                  ‰| «      dt	        ‰«      dz
  «      «      }t        | ‰|dz
     z
  «      t        | ‰|   z
  «      k  r|dz
  S |S )Nr)   )Úintr   ÚclipÚsearchsortedr4   Úabs)ÚxÚiÚgridss     €r&   Úfind_nearest_indexz1_discrete_line_search.<locals>.find_nearest_index…   sb   ø€ Ü”—‘œŸ™¨¨qÓ1°1´c¸%³jÀ1±nÓEÓFˆÜ˜A  a¨!¡e¡Ñ,Ó-´°A¸¸a¹±LÓ0AÒAˆq�1‰uÐHÀqÐHr(   c                ó‚   •— ‰j                  | «      }|�|S ‰|    ‰‰<   t        ‰j                  ‰«      «       }|‰| <   |S )N)ÚgetÚfloatrJ   )r\   Ú	cache_valÚnegvalr#   r]   Únegative_fval_cacheÚnormalized_paramsrN   s      €€€€€r&   Únegative_acqf_with_cachez7_discrete_line_search.<locals>.negative_acqf_with_cache�   sY   ø€ à'×+Ñ+¨AÓ.ˆ	ØÐ ØÐØ',¨Q¡xÐ˜)Ñ$ô ˜×.Ñ.Ð/@ÓAÓBÐBˆØ!'Ð˜AÑØˆr(   c           	     ó6  •— | ‰d   k  s| ‰d   kD  rt         j                  S t        t        j                  t        j                  ‰| «      dt        ‰«      dz
  «      «      }|dz
  } ‰|«       ‰|«      }}‰|   | z
  ‰|   ‰|   z
  z  }d|z
  }||z  ||z  z   S )Nr   éÿÿÿÿr)   g      ð?)r   ÚinfrW   rX   rY   r4   )	r[   ÚrightÚleftÚneg_acqf_leftÚneg_acqf_rightÚw_leftÚw_rightr]   rf   s	          €€r&   Úinterpolated_negative_acqfz9_discrete_line_search.<locals>.interpolated_negative_acqfœ   s«   ø€ Øˆu�Q‰xŠ<˜1˜u R™yš=Ü—6‘6ˆMÜ”B—G‘GœBŸO™O¨E°1Ó5°q¼#¸e»*Àq¹.ÓIÓJˆØ�q‰yˆá$ TÓ*Ù$ UÓ+ð &ˆð ˜‘, Ñ" u¨U¡|°e¸D±kÑ'AÑBˆØ˜‘,ˆØ˜Ñ%¨°.Ñ(@Ñ@Ð@r(   gê-�™—q=r   rh   Úbrent)ÚbracketÚmethodr=   T)r[   ra   r2   rW   )r\   rW   r2   ra   )r[   ra   r2   ra   )r4   r   Úiscloser7   ÚsoÚminimize_scalarr[   )r#   rL   rM   rN   r]   Úxtolr^   Úcurrent_choice_irp   ÚEPSÚresÚopt_idxÚfval_optrf   rd   re   s   `  ``        @@@r&   Ú_discrete_line_searchr}   y   s  ý€ ô ˆ5ƒz�Q‚à˜|¨UÐ2Ð2õIñ *¨.¸Ñ*CÓDÐÜ�:‰:�n YÑ/°Ð7GÑ1HÔIÐIÐIà+¨l¨]Ð;Ðà&×+Ñ+Ó-Ð÷
ñ 
öAð €CÜ
×
Ñ
Ø"ð �q‘˜C‘ Ð'7Ñ!8¸%À¹)Àc¹/ÐJØØô€Cñ ! §¡Ó'€GÙ(¨Ó1Ð1€Hð Ð"Ò" x°,Ò'>Ø',¨W¡~Ð˜)Ñ$Ø  (¨DÐ0Ð0à˜<¨Ð.Ð.r(   c                ó–   — d}| j                   j                  |   }|st        |«      |k  rt        | ||||«      S t	        | |||||«      S )Né   )Úsearch_spaceÚis_categoricalr4   rT   r}   )r#   rL   rM   rN   rO   rw   Ú MAX_INT_EXHAUSTIVE_SEARCH_PARAMSr�   s           r&   Ú_local_search_discreterƒ   ½   sZ   € ð (*Ð$à×&Ñ&×5Ñ5°iÑ@€NÙœ˜W›Ð)IÒIÜ! $¨¸ÀiÐQXÓYÐYä$ T¨>¸<ÈÐT[Ð]aÓbÐbr(   c           	     ó  — |j                  «       }|j                  «       }t        j                  t        |«      t        ¬«      }t        |«      D ]+  \  }	}
t        | |
||	   |||«      \  }}}|||	<   |||	<   |||	<   Œ- |||fS )Nr   )r7   r   r5   r4   r6   Ú	enumeraterƒ   )r#   r;   r<   rN   rO   rw   Úbest_normalized_params_batchedÚ
best_fvalsrE   Úbatchre   Úbest_normalized_paramsÚ	best_fvalÚupdateds                 r&   Ú_local_search_discrete_batchedrŒ   Ð   sª   € ð &<×%@Ñ%@Ó%BÐ"Ø×#Ñ#Ó%€Jä—x‘x¤ MÓ 2¼$Ô?ÐÜ$-Ð.DÓ$Eò *Ñ ˆÐ Ü5KØÐ# Z°Ñ%6¸	À7ÈDó6
Ñ2Ð 	¨7ð 1GÐ& uÑ-Ø%ˆ
�5ÑØ")Ð˜Òð*ð *¨:Ð7GÐGÐGr(   g-Cëâ6?éd   )r=   Úmax_iterc          
     ó  — | j                   | j                  j                  x}   }| j                  j                  }| j                  j	                  «       }|D �cg c]=  }t        j                  t        j                  |«      t
        j                  ¬«      dz  ‘Œ? }	}| j                  |j                  «       x}
«      }d}t        j                  t        |
«      |t        ¬«      }t        j                  t        |
«      «      }t        |«      D ]Ô  }t!        | |
|   ||   |||«      \  |
|<   ||<   }t        j"                  |||«      }t%        |||	«      D ]e  \  }}}|||k(  x}    }||    }|j&                  dk(  r|
|fc c S t)        | |
|   ||   |||«      \  |
|<   ||<   }t        j"                  |||«      }Œg |||k(  x}    }||    }|j&                  dk(  sŒÐ|
|fc S  t*        j-                  d«       |
|fS c c}w )N)Úinitialé   rh   r   r   z2local_search_mixed: Local search did not converge.)Úlength_scalesr€   r$   Údiscrete_indicesÚget_choices_of_discrete_paramsr   ÚminÚdiffri   rJ   r7   Úfullr4   rW   ÚarangeÚrangerF   r:   ÚzipÚsizerŒ   Ú_loggerÚwarning)r#   Úxs0r=   rŽ   Ú	cont_indsr%   r“   Úchoices_of_discrete_paramsrO   Údiscrete_xtolsÚbest_xsr‡   Ú
CONTINUOUSÚlast_changed_dimsÚremaining_indsÚ_r‹   r\   rw   Úis_convergeds                       r&   Úlocal_search_mixed_batchedr¨   è   sE  € ð ×%Ñ%°D×4EÑ4E×4XÑ4XÐ'X yÑZ€LØ×(Ñ(×9Ñ9ÐØ!%×!2Ñ!2×!QÑ!QÓ!SÐð
 2ö	ð ô 	�‰Œr�w‰w�wÓ¬¯©Ô0°1Ó4ð€Nð ð ×'Ñ'°C·H±H³JÐ)>¨Ó@€JØ€JÜŸ™¤ G£¨jÄÔDÐÜ—Y‘Yœs 7›|Ó,€NÜ�8‹_ò NˆÜG_Ø�'˜.Ñ)¨:°nÑ+EÀyÐR^Ð`cóH
ÑDˆ�Ñ ¨NÑ!;¸Wô ŸH™H W¨jÐ:KÓLÐÜ #Ð$4Ð6PÐR`Ó aò 
	HÑˆAˆw˜Ø 1ÐDUÐYZÑDZÐ4Z°LÐ2[Ñ \ÐØ+¨\¨MÑ:ˆNØ×"Ñ" aÒ'Ø 
Ð*Ô*ä.Ø˜' .Ñ1°:¸nÑ3MÈqÐRYÐ[_óñ IˆG�NÑ# Z°Ñ%?Àô
 !#§¡¨°!Ð5FÓ GÑð
	Hð (Ð:KÈzÑ:YÐ*Y¨,Ð(ZÑ[ˆØ-¨|¨mÑ<ÐØ×Ñ !Ó#Ø˜JÐ&Ò&ð+Nô. 	�‰ÐLÔMØ�JÐÐùòEs   ÁAG?i   é
   )Ú!warmstart_normalized_params_arrayÚn_preliminary_samplesÚn_local_searchr=   Úrngc               óö  — |xs t         j                  j                  «       }|€+t        j                  d| j                  j
                  f«      }t        |«      |dz
  k  sJ d«       ‚| j                  j                  ||¬«      }| j                  |«      }t        |t         j                  «      sJ ‚t        j                  |«      }t        j                  |||   z
  «      }	d|	|<   |	|	j                  «       z  }	t        t        j                  |	dkD  «      «      }
t!        |t        |«      z
  dz
  |
«      }||
k(  rt"        j%                  d«       t        j&                  |g«      }|dkD  r4|j)                  t        |«      |d|	¬«      }t        j*                  ||«      }t        j,                  ||d d …f   |g«      }t/        | ||¬	«      \  }}t        j                  |«      j1                  «       }||   ||   fS )
Nr   r)   zPWe must choose at least 1 best sampled point + given_initial_xs as start points.)r­   g        zBStudy already converged, so the number of local search is reduced.F)r›   ÚreplaceÚp)r=   )r   ÚrandomÚRandomStateÚemptyr€   Údimr4   Úsample_normalized_paramsrJ   Ú
isinstanceÚndarrayrK   Úexpr   rW   Úcount_nonzeror•   rœ   r�   r   ÚchoiceÚappendÚvstackr¨   Úitem)r#   rª   r«   r¬   r=   r­   Ú
sampled_xsÚf_valsÚmax_iÚprobsÚn_non_zero_probs_improvementÚn_additional_warmstartÚchosen_idxsÚadditional_idxsÚx_warmstartsr¢   r‡   rS   s                     r&   Úoptimize_acqf_mixedrÇ     sã  € ð Ò
(”—‘×&Ñ&Ó(€Cà(Ð0Ü,.¯H©H°a¸×9JÑ9J×9NÑ9NÐ5OÓ,PÐ)äÐ0Ó1°^ÀaÑ5GÒGð ØZóÐGð ×"Ñ"×;Ñ;Ð<QÐWZÐ;Ó[€Jð ×#Ñ# JÓ/€FÜ�fœbŸj™jÔ)Ð)Ð)ä�I‰I�fÓ€Eô
 �F‰F�6˜F 5™MÑ)Ó*€EØ€Eˆ%�LØ	ˆU�Y‰Y‹[Ñ€EÜ#&¤r×'7Ñ'7¸À¹Ó'DÓ#EÐ ä ØœÐ>Ó?Ñ?À!ÑCÐEaóÐð Ð!=Ò=Ü�‰Ð\Ô]Ü—(‘(˜E˜7Ó#€KØ Ò!ØŸ*™*Ü�
‹OÐ"8À%È5ð %ó 
ˆô —i‘i ¨_Ó=ˆä—9‘9˜j¨²a¨Ñ8Ð:[Ð\Ó]€LÜ4°T¸<ÈSÔQÑ€GˆZÜ�y‰y˜Ó$×)Ñ)Ó+€HØ�8Ñ˜j¨Ñ2Ð2Ð2r(   )r#   r   r;   r1   r<   r1   r$   r1   r%   r1   r=   ra   r2   ú)tuple[np.ndarray, np.ndarray, np.ndarray])r#   r   rL   r1   rM   ra   rN   rW   rO   r1   r2   útuple[np.ndarray, float, bool])r#   r   rL   r1   rM   ra   rN   rW   r]   r1   rw   ra   r2   rÉ   )r#   r   rL   r1   rM   ra   rN   rW   rO   r1   rw   ra   r2   rÉ   )r#   r   r;   r1   r<   r1   rN   rW   rO   r1   rw   ra   r2   rÈ   )
r#   r   rž   r1   r=   ra   rŽ   rW   r2   r3   )r#   r   rª   znp.ndarray | Noner«   rW   r¬   rW   r=   ra   r­   znp.random.RandomState | Noner2   ztuple[np.ndarray, float])Ú
__future__r   r8   Útypingr   r   r   Ú"optuna._gp.scipy_blas_thread_patchr   Úoptuna.loggingr   Úscipy.optimizeÚoptimizeru   r
   Ú
optuna._gpr   Úoptuna._gp.acqfr   Úoptunar	   Ú__name__rœ   rF   rT   r}   rƒ   rŒ   r¨   rÇ   © r(   r&   ú<module>rÕ      se  ðÝ "ã Ý  ã å YÝ %ñ ÝÛå)Þ3å"á	Ð%Ó	&€BÙ˜Ó €EÙ Ð!<Ó=€Nñ �XÓ
€ðAØ
ðAà&ðAð ðAð #ð	Að
 ðAð 
ðAð /óAðH/Ø
ð/àð/ð ð/ð ð	/ð
 ð/ð $ó/ð0A/Ø
ðA/àðA/ð ðA/ð ð	A/ð
 ðA/ð ðA/ð $óA/ðHcØ
ðcàðcð ðcð ð	cð
 ðcð ðcð $ócð&HØ
ðHà&ðHð ðHð ð	Hð
 ðHð ðHð /óHð2 AEÐVYñ-Ø
ð-Ø$.ð-Ø8=ð-ØPSð-à"ó-ðf <@Ø!%ØØØ(,ñ13Ø
ð13ð (9ð13ð ð	13ð
 ð13ð 
ð13ð 
&ð13ð ô13r(   