
    @ۀh)H                     <   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
mZmZmZ d dlmZ d dlmZmZ d dlmZ d dlmZmZ d dlZd dlmZmZmZmZ d dlZd d	lmZ  e	       Z e         ej@                  d
      Z!ejE                  dedgd       e
dddg      fdee   fd       Z#ejI                  dedgd      d        Z%ejI                  dedgd      d        Z& ejN                  d       e( ejN                  dd             ejN                  d       ejN                  d       ejN                  d       d!d"Z) ejN                  d#      Z*d$d d d%Z+ G d& d'e      Z,ejE                  d(d)gd*e,+      defd,       Z-ded-e.fd.Z/d/e.d0e.fd1Z0dFd2e.d3e(d0e(fd4Z1i Z2e3e4e.e(f   e(f   e5d5<   d2e.d6e(d0e(fd7Z6d8e7e(   d9e7e(   fd:Z8d8e7e(   d;e(fd<Z9d$d d d%Z+ejE                  d=d)gd>?      d@efdA       Z:dB Z;ejI                  dCd)gdD?      dE        Z<y)G    N)load_dotenv)	APIRouterBodyQueryBackgroundTasksDepends)JSONResponse)HttpUrl	BaseModel)List)datetimetimezone)AddLinksResponseStatusResponseParsedResultsResponseFilterPayload)tqdmzuvicorn.errorz/parser/add-linksu   Парсингu   Додати лінк)response_modeltagssummary.zhttps://example.com/page1zhttps://example.com/page2)examplepayloadc           
        K   t         j                  }d}|j                         4 d {   }|j                         4 d {   }| D ]O  }|j	                  dt        |      t        j                  t        j                        ddf       d {    |dz  }Q d d d       d {    d d d       d {    |ddS 7 7 7 47 # 1 d {  7  sw Y   /xY w7 &# 1 d {  7  sw Y   6xY ww)Nr   zPINSERT INTO url_queue (url, date_added, status, recived) VALUES (%s, %s, %s, %s)   success)addedstatus)
statedb_poolacquirecursorexecutestrr   nowr   utc)r   pooladded_countconnr!   urls         /var/www/html/api.py	add_linksr+      s      ==DK||~ ! !;;= 	! 	!F !nnfXx||HLL91a@   q !	! 	!! ! !I66!	!	! 	! 	! 	!! ! ! !s   'C4C C4CCCA	CC
CC&C'C+C46C7
C4CCCC	CC	CC4C1%C(&C1-C4z/parser/statusu   Отримати статусc                    K   t         j                  } | j                         4 d {   }|j                  t        j
                        4 d {   }|j                  d       d {    |j                          d {   d   }|j                  d       d {    |j                          d {   d   }|j                  d       d {    |j                          d {   d   }d d d       d {    d d d       d {    t         j                  j                  dS 7 7 7 7 7 7 7 k7 U7 D# 1 d {  7  sw Y   TxY w7 K# 1 d {  7  sw Y   [xY ww)NzHSELECT COUNT(*) as count FROM url_queue WHERE status = 2 AND recived = 0countz8SELECT COUNT(*) as count FROM url_queue WHERE status = 0z8SELECT COUNT(*) as count FROM url_queue WHERE status = 1)processed_since_last	new_linksin_queueparser_status)
r   r   r    r!   aiomysql
DictCursorr"   fetchoner1   value)r&   r(   r!   	processedr/   r0   s         r*   
get_statusr7   /   s>    ==D||~ : :;;x223 	: 	:v..!klll%00':I..![\\\%00':I..![\\\$oo//9H	: 	:: : !*,,22	 :	:l0\0\/	: 	: 	: 	:: : : :s   %E.D/E.$ED2EE(D4)E D6ED8E4D:5ED<E(D>)E0E;E <E E.E$E.2E4E6E8E:E<E>E EE	E	E	EE.E+E" E+'E.z/parser/parsed-resultsu#   Отримати результатc                  j  K   t         j                  } | j                         4 d {   }|j                  t        j
                        4 d {   }|j                  d       d {    |j                          d {   }|j                  d       d {    |j                          d {   }dd l}|D ]A  }|j                  d      st        |d   t              s)	  |j                  |d         |d<   C |D ]A  }|j                  d      st        |d   t              s)	  |j                  |d         |d<   C |rd|D cg c]  }|d   	 }}|j                  ddj                  dgt        |      z         d	|       d {    |j!                          d {    |rd|D cg c]  }|d   	 }}|j                  ddj                  dgt        |      z         d	|       d {    |j!                          d {    d d d       d {    d d d       d {    d
S 7 7 7 7 7 7 # t        $ r	 d |d<   Y w xY w# t        $ r	 d |d<   Y mw xY wc c}w 7 7 c c}w 7 7 u7 g# 1 d {  7  sw Y   wxY w7 n# 1 d {  7  sw Y   ~xY ww)Nz8SELECT * FROM url_queue WHERE status = 2 AND recived = 0z8SELECT * FROM url_queue WHERE status = 3 AND recived = 0r   dataurl_idz2UPDATE url_queue SET recived = 1 WHERE url_id IN (,%s))parsederror)r   r   r    r!   r2   r3   r"   fetchalljsonget
isinstancer#   loads	Exceptionjoinlencommit)	r&   r(   r!   status_2_rowsstatus_3_rowsrA   rowidsids_3s	            r*   get_parsed_resultsrN   C   s    ==D||~ '$ '$;;x223 &	$ &	$v..![\\\"(//"33M..![\\\"(//"33M $ +776?z#f+s'C+&0djjV&=F+ % +776?z#f+s'C+&0djjV&=F+ 0=>s8}>>nnHSWRXY\]`YaRaIbHccde   kkm##2?@3X@@nnHSWRX[^_d[eReIfHgghi   kkm##M&	$ &	$'$ '$T   S'$&	$\3\3 % +&*F+ % +&*F+
 ? $ A $M&	$ &	$ &	$ &	$'$ '$ '$ '$s  %J3H7J3$JH:JJ(H=)J I JIJ1I2JJ&I	>JJ,IJI37JI8J'I:(
J2I<>7J5J6JJJJJJ"J3-J.
J3:J=J JJJ	I	JI	JI0	,J/I0	0	J:J<JJJJ	JJ	JJ3J0$J'%J0,J3DB_HOST_FILTERDB_PORT_FILTERi  DB_USER_FILTERDB_PASSWORD_FILTERDB_NAME_FILTERT)hostportuserpassworddb
autocommit
SECRET_KEYF)runningtotalr6   c                   ,    e Zd ZU eed<   eed<   eed<   y)FilterResponser9   r?   r   N)__name__
__module____qualname__r#   __annotations__bool     r*   r^   r^      s    
IJLre   r^   z/filter/filtersu   Фільтрu%   Завантажити фільтра)r   r   r   c                    K   t        j                  t        j                        j	                  d      }t        | |       d {    ddddS 7 
w)N%Y-%m-%d %H:%M:%Su"   Фільтри оброблені. T)r9   r?   r   )r   r$   r   r%   strftimeprocess_filter_payload)r   r$   s     r*   
parse_datark      sL     
,,x||
$
-
-.A
BC
 #
...4  /s   AAAA	timestampc                 h  K   t        j                  di t         d {   }|j                  t         j                         d {   }| j
                  }| j                  }g }i }t               }| j                  D ]Z  }	|	j                  }
|	j                  xs d}|	j                  }t        ||
|       d {   }|
||<   |j                  ||||d       \ i }| j                  D ]t  }|j                  }t!        ||j                  |       d {   }|j#                  |       |j                  |j%                  |j                  d      d||j                  <   v i }| j&                  D ]G  }|j)                  |j*                  g       j                  |j%                  |j,                  i              I g }g }t/        |j1                               }|rc|j3                  ddj5                  dgt7        |      z         d|       d {    |j9                          d {   D ch c]  }|d	   	 }}n
t               }|j;                         D ]L  \  }}t=        j>                  |d
      }||v r|j                  ||||f       7|j                  |||||f       N d}tA        dt7        |      |      D ]A  }||||z    }|jC                  d|       d {    tE        jF                  d       d {    C tA        dt7        |      |      D ]A  }||||z    }|jC                  d|       d {    tE        jF                  d       d {    C |D ]$  }|j3                  d|d   |d   f       d {    & |jC                  d|D cg c]  }|d   |d   |d   |d   f c}       d {    |jI                          d {    |jK                          d {    |jM                          d {    y 7 7 7 P7 7 7 c c}w 7 ?7 %7 7 7 c c}w 7 u7 _7 I7 3w)Nr   )filter_group_idcategory_idcategory_name
sort_orderrh   )filter_namefilter_groupz@SELECT product_id FROM xpro_recived_filter WHERE product_id IN (r;   r<   r=   
product_idF)ensure_asciii'  zyINSERT INTO xpro_recived_filter (product_id, filters, date_added, date_update, category_name) VALUES (%s, %s, %s, %s, %s)gMbP?zgUPDATE xpro_recived_filter SET filters = %s, date_update = %s, category_name = %s WHERE product_id = %szVDELETE FROM xpro_category_filter_group WHERE category_id = %s AND filter_group_id = %sro   rn   zxINSERT INTO xpro_category_filter_group (category_id, category_name, filter_group_id, sort_order) VALUES (%s, %s, %s, %s)rp   rq   rd   )'r2   connectDB_CONFIG_FILTERr!   r3   Category_nameCategory_idsetFilterGroupsnamerq   idget_or_create_filter_groupappendFiltersgroup_idget_or_create_filteraddrB   Products
setdefaultrt   	filter_idlistkeysr"   rF   rG   r@   itemsrA   dumpsrangeexecutemanyasynciosleeprH   closeensure_closed)r   rl   r(   r!   categoryro   order_filtersall_filter_groupsunique_filter_idsgroupr|   rq   r   rn   all_filtersfr   productsp	to_insert	to_updateproduct_idsrK   existing_productsrt   filtersfilters_json
BATCH_SIZEibatchs                                 r*   rj   rj      sd    !!5$455D;;x2233F$$H%%KM%% zz%%*
88 :64 TT&*(#.&%$	
 	 K__ 
**.vqvvOO	i(66-11!**bA
ADD	
 H WALL"-44[__Q[[RT5UVW IIx}}'KnnNsxxY]X^adepaqXqOrNsstu
 	
 	
 AG@Q:QR3S.RRE'~~/ Y
Gzz'>**lIxLMj,	9hWXY J 1c)nj1 #!A
N+   H
 	
 	
 mmE"""# 1c)nj1 #!A
N+  u
 	
 	
 mmE"""#  
nnd}q!234
 	
 	

 

 	C`mn[\!M
Ao.2C0Dao	Vn  
 ++-
,,.



s 63 U P 	
 ;RR 	
 	#
	
 	#	
 	o
 s'  P2P&P2PA1P28P9AP2PC9P2PP2#P$P2*P6B$P2PP27P8<P24P5P2P!(P2:P#;P2P%
*
P24P*5P2P,P2$P.%P2<P0=P2P2P2P2P2P2P2P2P2!P2#P2%P2,P2.P20P2textreturnc                    i ddddddddd	d
dddddddddddddddddddddddd i d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAd<i dBdCdDd<dEddFddGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d^i d`d\dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~ddi dd<dddddddd<dd\ddTdddddd<dd<dddd<dd<dd<dd<dd<d<d<d<d<d<d<dd<d<d<d<ddd<d<dd<j                  fd| D              } | j                         } t        j                  dd<|       } t        j                  dd|       } d| j	                  d      z   S )Nu   аau   бbu   вvu   гhu   ґgu   дdu   еeu   єieu   жzhu   зzu   иyu   іr   u   їu   йu   кku   лlu   мmu   нnu   оou   пr   u   рru   сsu   тtu   уuu   фr   u   хkhu   цtsu   чchu   шshu   щshchu   ьrh   u   юiuu   яiau   ʼu   ёyou   ъu   ыu   эu   АAu   БBu   ВVu   ГHu   ҐGu   ДDu   ЕEu   ЄYeu   ЖZhu   ЗZu   ИYu   ІIu   Їu   Йu   КKu   ЛLu   МMu   НNu   ОOu   ПPu   РRu   СSu   ТTu   УUu   ФFu   ХKhu   ЦTsu   ЧChu   ШShu   ЩShchu   Ьu   ЮYuu   ЯYau   ЁYou   Ъu   Ыu   Э -_r;   ./:;?!#and)(r=   "'   «   »&%@=+u   –u   —u   №u   ’c              3   B   K   | ]  }j                  ||        y w)N)rB   ).0cmap_s     r*   	<genexpr>ztranslit.<locals>.<genexpr>   s     0a488Aq>0s   z[^a-z0-9\-]z-+f-)rF   lowerresubstrip)r   r   s    @r*   translitr      s   Scs#',059#>B3GKDQUVZ[_`cdhilmqruvz{~ @D  EH IM  NQ RV  WZ [_  `c dh  il mq  ru vz  {~ C  DG HL  MP QU  VY Z^  _b cg  hk lp  qu vz  { @D  EI JN  OS TX  Y_ `d  eg hl  mq rv  w{ |@  ACTrs#' 	S c s $( -1 6:# ?C3 HLD RVVZ \``c eiil nrru w{{~ @D  EH IM  NQ RV  WZ [_  `c dh  il mq  ru vz  {~ C  DG HL  MP QU  VY Z^  _b cg  hk lp  qu vz  { @D  EI JN  OS TX  Y_ `d  eg hl  mq rv  w{ |@  AE FJ  KM NR  SV W[  \_ 	C	 C	 B	  #2	 '*#	 /2"	 69	 =@	 DGr	 KNb	 VX\^cekmsu{}  CH  MO  TV  []  bd  kn  ux  A  HJ	D 770400D::<D66."d+D66%d#D$**S/!!re   r|   rq   c                   K   |sd}| j                  d|f       d {    | j                          d {   }|sl| j                  d|f       d {    | j                  }| j                  d||f       d {    t        |      }| j                  d||f       d {    |S |d   }| j                  d||f       d {    |S 7 7 7 |7 V7 17 w)Ni z9SELECT * FROM oc_filter_group_description WHERE name = %sz4INSERT INTO oc_filter_group (sort_order) VALUES (%s)z_INSERT INTO oc_filter_group_description (filter_group_id, language_id, name) VALUES (%s, 2, %s)zLINSERT INTO xpro_filter_group_seo (filter_group_id, keyword) VALUES (%s, %s)rn   zEUPDATE oc_filter_group SET sort_order = %s WHERE filter_group_id = %s)r"   r4   	lastrowidr   )r!   r|   rq   resultrn   keywords         r*   r~   r~     s	    

..TW[V]
^^^??$$FnnSV`Ubccc **nnmd#
 	
 	
 4.nnZg&
 	
 	
  !23nnS)
 	
 	
 - _$ 	d	

	
	
sg   CCCCCC'C:C;&C!C""CCCCCCCCfilter_cachern   c           	        K   ||f}|t         v r	t         |   S | j                  d||f       d {    | j                          d {   }|r|d   t         |<   |d   S | j                  d|df       d {    | j                  }| j                  d|||f       d {    t	        |      }|j                  dd      }| j                  d||||||f       d {    |t         |<   |S 7 7 7 y7 R7 w)	NzTSELECT filter_id FROM oc_filter_description WHERE name = %s AND filter_group_id = %sr   zCINSERT INTO oc_filter (filter_group_id, sort_order) VALUES (%s, %s)r   zhINSERT INTO oc_filter_description (filter_id, language_id, filter_group_id, name) VALUES (%s, 2, %s, %s)r   zf-not-zwINSERT INTO xpro_filter_seo (filter_id, type, keyword, filter_group_id) VALUES (%s, 'add', %s, %s), (%s, 'sub', %s, %s))r  r"   r4   r  r   replace)r!   r|   rn   keyr  r   r  keyword_subs           r*   r   r   !  s9    
!C
lC  
..^	   ??$$F";/Sk""
..M	!     I
..r	OT*  
 tnG//$1K
.. 	B	G_ioV  
 "L= %sW   -C6C,C6C..C66C07(C6C2 <C6C4C6.C60C62C64C6
filter_idscategory_idsc                    K   g }|D ]  }|D ]  }|j                  ||f         |D ]  }	 | j                  d|       d {     y 7 #  Y 'xY ww)NzGINSERT INTO oc_category_filter (category_id, filter_id) VALUES (%s, %s))r   r"   )r!   r
  r  insert_valuesr   ro   r5   s          r*   add_filter_categoriesr  G  s|     M ;	' 	;K  +y!9:	;;  	..Y  	s.   )AAA	AA	AAArt   c                    K   |D cg c]  }||f }}|D ]  }	 | j                  d|       d {     y c c}w 7 #  Y ,xY ww)NzEINSERT INTO oc_product_filter (product_id, filter_id) VALUES (%s, %s))r"   )r!   r
  rt   r   r  r5   s         r*   add_product_filterr  W  sb     >HIj),IMI 	..W   J	s.   A7A><>A>A Az/process-filtersu2   Запустити обробку фільтрів)r   r   background_tasksc                 Z   K   t         d   rdddS | j                  t               dddS w)Nr[   Fu&   Обробка вже запущена)r   messageTu<   Обробка фільтрів запущена у фоні)processing_statusadd_taskprocess_filters_loop)r  s    r*   start_process_filtersr  j  s5     #,TUU23'effs   )+c                    K   dt         d<   dt         d<   dt         d<   t        j                  di t         d {   } | j	                  t        j
                         d {   }	 |j                  d       d {    |j                          d {   }d}|r	d|v r|d   }|j                  d	       d {    |j                          d {   }t        |      t         d<   t        t         d   d
      }|D cg c]  }|d   	 }}|r<ddj                  dgt        |      z         d}|j                  ||       d {    i }	i }
t        |d      D ]  \  }}|d   }t        j                  |d         }g }|D ]  }|j                  d      }|j                  d      }|r|st!        d|        8|j#                         }||	vrt%        ||       d {   }||	|<   n|	|   }|j#                         |f}||
vrt'        |||       d {   }||
|<   n|
|   }|j)                  |        d}|j                  ||f       d {    |j                          d {   }|D cg c]  }|d   	 }}t+        |||       d {    t-        |||       d {    t/        j0                  t2        j4                        j7                  d      }|j                  d||f       d {    |t         d<   |j9                  d        | j;                          d {    dt         d<   |j=                          d {    | j=                          y 7 %7 7 7 7 7 c c}w 7 /7 7 f7 .7 c c}w 7 7 7 7 g7 H# dt         d<   |j=                          d {  7   | j=                          w xY ww)NTr[   r   r6   r\   zISELECT * FROM xpro_process_filter ORDER BY process_filter_id DESC LIMIT 1z2000-01-01 00:00:00date_processz
            SELECT * FROM xpro_recived_filter
            WHERE date_update > IFNULL(date_process, '2000-01-01 00:00:00')
        u%   Опрацювання товарів)r\   descrt   z3DELETE FROM oc_product_filter WHERE product_id IN (r;   r<   r=   r   )startr   rs   rr   un   Warning: пропускаємо фільтр без імені групи або фільтра у product_id=zDSELECT category_id FROM oc_product_to_category WHERE product_id = %sro   rg   zFUPDATE xpro_recived_filter SET date_process = %s WHERE product_id = %sFrd   )r  r2   rv   rw   r!   r3   r"   r4   r@   rG   r   rF   	enumeraterA   rD   rB   printr   r~   r   r   r  r  r   r$   r   r%   ri   updaterH   r   )r(   r!   last_processdateproducts_resultbarr   r   
sql_deletefilter_group_cacher  r   productrt   r   r
  
filter_objfilter_group_namerr   fg_keyrn   f_keyr   sqlcategories_resultrK   
categories
date_addeds                               r*   r  r  r  s    #'i %&k"!"g!!5$455D;;x2233FPnnhiii#__..$Nl:/Dnn   	 	 !' 11%(%9'"*73:ab0?@1q@@NsxxY]X^adepaqXqOrNsstuJ..[999#O1= 1	JAw .Jjj!34GJ% -
$.NN>$B!(nn]; )  K  LV  KW  X  Y*002!33,FvO`,a&aO1@&v.&8&@O$**,o>,&:6;P_&` `I*3L' ,U 3I!!),/-2 YC..zm444&,oo&7 78IJ#m,JJJ (
JGGG$VZDDD!hll3<<=PQJ..!ilv  yC  lD  E  E  E-.k*JJqMc1	h kkm',)$lln

g 63 	j.
	 2
 A :* 'b !a 5 7J HD E 	 	 (-)$lln

s[  8OM&O!M""O'N ;M%<N M(%N 9M+:N M..N  M1;N M6BN  M9!5N M<:N M?N )N*N 2N>N N
N %N&AN 3N44N (N)N -O	N
O"O%N (N +N .N 1N 9N <N ?N N N N N N OO1N42OOz/processing-statusu,   Статус обробки фільтрівc                     K   t         d   t         d   t         d   t         d   dkD  r"t        t         d   t         d   z  dz  d      dS ddS w)Nr[   r6   r\   r   d      )r[   r6   r\   percent)r  roundrd   re   r*   get_processing_statusr3    sj      %Y/&{3"7+W%) +K8;LW;UUY\\^_`	 
 01 s   AA)r   )=r   rA   osr   timer2   dotenvr   fastapir   r   r   r   r   fastapi.responsesr	   pydanticr
   r   typingr   r   r   r   schemasr   r   r   r   loggingr   router	getLoggerloggerpostr+   rB   r7   rN   getenvintrw   rZ   r  r^   rk   r#   rj   r   r~   r  dicttuplerb   r   r   r  r  r  r  r3  rd   re   r*   <module>rE     s     	 	    D D * '  '  Z Z  	  
		?	+ 1AIYHZd{|!''
7']7 }7, ^CSBT^}~ & $5JRbQc  nS  T. T.h BII&'			*D12BII&'		./
"))$
%  RYY|$
  Y 
 n%5?f  xF  Gm  GZ- ZC Zx"3 "3 "3 C PS : ,.d5c?C'( -#S #3 #3 #LDI TRUY  
c 
 
   ~&6@tug/ g vgXv  '7Aop qre   