
    Qi"                     R   S SK r S SKJr  S SKJr  S SKJr  S SKrS SKJrJ	r	  S SK
JrJr  S SKJr  S SKJr  \" S	S
/S9r\R&                  " S\S9\R(                  " SSSSS9\R(                  " SSSSS9\R(                  " SSSSS9\R(                  " SSSSSS9\R*                  " 5       S  5       5       5       5       5       5       r\" \R0                  R3                  S!5      S    5      S":  a  \R4                  O\R6                  r\" 5       S# 5       rS$ rS% r\R?                  S&5      \R(                  " S'S(SSS)S*9\R(                  " S+S,SSS-S*9\R(                  " S.S/S0S19\R(                  " S2S3S4S19\S5 5       5       5       5       5       5       r \R?                  S65      \R(                  " S7S,SSS8S*9\S9 5       5       5       r!\R?                  S:5      \R(                  " S;SSSS<S*9\R(                  " S=S>SSS?S*9\R(                  " S@S/SSSAS*9\R(                  " SBS3SSSCS*9\SD 5       5       5       5       5       5       r"\R?                  SE5      \R(                  " SFSGSSHSI9\R(                  " SJS(SSSKS*9\R(                  " SLS/SSSMS*9\SN 5       5       5       5       5       r#\R?                  SO5      \R(                  " SFSGSSPSI9\R(                  " SJS(SSSKS*9\R(                  " SLS/SSSQS*9\SR 5       5       5       5       5       r$\R?                  SS5      \R(                  " STS(SSSUS*9\SV 5       5       5       r%g)W    N)deepcopypartialupdate_wrapper)MosesTokenizerMosesDetokenizer)MosesTruecaserMosesDetruecaser)MosesPunctNormalizerparallelize_preprocessz-hz--help)help_option_namesT)chaincontext_settingsz
--languagez-lenz+Use language specific rules when tokenizing)defaulthelpz--processesz-j   zNo. of processes.z
--encodingz-eutf8zSpecify encoding of file.z--quietz-qFzDisable progress bar.)is_flagr   r   c                     g N )languageencoding	processesquiets       `/var/www/html/backend/Backoffice_Marketplace/venv/lib/python3.13/site-packages/sacremoses/cli.pyclir       s     	    .   c                     [         R                  " SUS9 nUnU  H  nU" [        U5      40 UD6nM     U(       a  U H  n[         R                  " U5        M     S S S 5        g ! , (       d  f       g = f)Nstdinr   )clickget_text_streamlistecho)
processorsr   kwargsfiniteratorprocitems          r   process_pipeliner1   $   s[    			w	:cDDN5f5H  

4  ! 
;	:	:s   AA##
A1c                 *   ^  U 4S jn[        UT 40 UD6$ )zTHelper decorator to rewrite a function so that
it returns another function from it.
c                  (   > U4S jn[        U40 U D6$ )Nc                    > T" U 40 UD6$ r   r   )streamr,   fs     r   	processor.processor.<locals>.new_func.<locals>.processor5   s    V&v&&r!   r   )r,   r7   r6   s     r   new_funcprocessor.<locals>.new_func4   s    	' y+F++r!   r   )r6   r,   r9   s   `  r   r7   r7   /   s    
, (A000r!   c              #   t   #    US:X  a  U  H  nU" U5      v   M     g [        XX#(       + S9 H  nUv   M	     g 7f)Nr   )progress_barr   )r.   funcr   r   lineoutlines         r   parallel_or_notr@   =   s?     A~Dt*  .IY
G M
s   68tokenizez--aggressive-dash-splitsz-azTriggers dash split rules.)r   r   r   z--xml-escapez-xz"Escape special characters for XML.z--protected-patternsz-pzXSpecify file with patters to be protected in tokenisation. Special values: :basic: :web:)r   z--custom-nb-prefixesz-czjSpecify a custom non-breaking prefixes file, add prefixes to the default ones from the specified language.c                 `   [        XS9nU(       af  US:X  a  UR                  nOSUS:X  a  UR                  nO@[        USS9 n	U	R	                  5        V
s/ s H  oR                  5       PM     nn
S S S 5        [        UR                  SUUUS9n[        XX#5      $ s  sn
f ! , (       d  f       N6= f)N)lang custom_nonbreaking_prefixes_filez:basic:z:web:r   r&   T)
return_straggressive_dash_splitsescapeprotected_patterns)	r   BASIC_PROTECTED_PATTERNSWEB_PROTECTED_PATTERNSopen	readlinesstripr   rA   r@   )r.   r   r   r   
xml_escaperF   rH   custom_nb_prefixesmosesr-   patternmoses_tokenizes               r   tokenize_filerS   M   s    H E *!&!?!?7*!&!=!=(6:cEH]]_%U_'mmo_"%U ; 5-N 8YFF &V ;:s   BB-BB
B-
detokenizez--xml-unescapez$Unescape special characters for XML.c                     [        US9n[        UR                  SUS9n[        [	        [        [        R                  U 5      5      XbU5      $ )N)rC   T)rE   unescape)r	   r   rT   r@   r)   mapstrsplit)r.   r   r   r   xml_unescaperP   moses_detokenizes          r   detokenize_filer\      sH      (+Eu//D<XSH%&(8U r!   	normalizez--normalize-quote-commasz Normalize quotations and commas.z--normalize-numbersz-dzNormalize number.z--replace-unicode-punctsz2Replace unicode punctuations BEFORE normalization.z--remove-control-charsz.Remove control characters AFTER normalization.c                 ^    [        UUUUUS9n[        UR                  5      n	[        X	X#5      $ )N)norm_quote_commasnorm_numberspre_replace_unicode_punctpost_remove_control_chars)r   r   r]   r@   )
r.   r   r   r   normalize_quote_commasnormalize_numbersreplace_unicode_punctsremove_control_charsrP   moses_normalizes
             r   normalize_filerh      s;    H !0&"8"6E eoo.O8iGGr!   ztrain-truecasez--modelfilez-mzFilename to save the modelfile.)requiredr   z--is-asrz)A flag to indicate that model is for ASR.z--possibly-use-first-tokenz*Use the first token as part of truecasing.c                 f    [        US9nUR                  U UUU(       + S9nUR                  U5        g )Nis_asrpossibly_use_first_tokenr   r<   )r
   train
save_model)	r.   r   r   r   	modelfilerl   rn   rP   models	            r   train_truecaserrs      s?    , &)EKK!9i	  E 
Yr!   truecasez$Filename to save/load the modelfile.z1Use the first token as part of truecase training.c                    [         R                  R                  U5      (       d<  [        U 5      n[	        US9nUR                  UUUU(       + S9n	UR                  U5        [	        XES9n
[        U
R                  SS9n[        XX#5      $ )Nrk   rm   )	load_fromrl   T)rE   )
ospathisfiler   r
   ro   rp   r   rt   r@   )r.   r   r   r   rq   rl   rn   iterator_copy	truecaserrr   rP   moses_truecases               r   truecase_filer}     s    . 77>>)$$ *"&1	%=#)	   
 	Y'Y>EU^^=N8YFFr!   
detruecasez--is-headlinezWhether the file are headlines.c                 X    [        5       n[        UR                  SUS9n[        XX#5      $ )NT)rE   is_headline)r   r   r~   r@   )r.   r   r   r   r   rP   moses_detruecases          r   detruecase_filer   /  s3     ET{ 8yHHr!   )&rw   copyr   	functoolsr   r   r'   sacremoses.tokenizer   r	   sacremoses.truecaser
   r   sacremoses.normalizer   sacremoses.utilr   dictCONTEXT_SETTINGSgroupoptionversion_optionr    int__version__rY   resultcallbackresult_callbackr1   r7   r@   commandrS   r\   rh   rs   r}   r   r   r!   r   <module>r      s   
   $  @ @ 5 24*:;  4*:;$+X mT13FGlD&7RStT57N 	  T H <	 ),E,=,=,C,CC,H,K(Lq(P#$$VYViVi! !1  Z	% 	- 	c
 	u
 G  4GL \	/   & [	+ 4tBU 	= 	9 H  4H4 4$-N 	4  	5    & & Z4$-S 	4  	< G  &G2 \	* I  Ir!   