Ë
    0�£j¡  ã                   óp   — d dl Z d dlZd dlZej                  r	d dlmZmZ nd dlmZmZ d„ Z G d„ d«      Z	y)é    N)ÚurlopenÚRequestc                 ó:   — t        dj                  | |«      «       y )NzDownloaded {}, Total is {})ÚprintÚformat)Ú
downloadedÚ
total_sizes     ú<C:\FundamentalWorkbench\vendor\pytdx\crawler\base_crawler.pyÚdemo_reporthookr      s   € Ü	Ð
&×
-Ñ
-¨j¸*Ó
EÕFó    c                   ó8   — e Zd Zdd„Zd	d„Zd	d„Zd„ Zd	d„Zd„ Zy)
ÚBaseCralwerc                 ó   — d| _         y )NÚhttp)Úmode)Úselfr   s     r
   Ú__init__zBaseCralwer.__init__   s	   € Øˆ�	r   Nc                 óÞ   — | j                   dk(  r | j                  |||||dœ|¤Ž}n | j                  ||||dœ|¤Ž} | j                  |g|¢­i |¤Ž}	 |j	                  «        |S #  Y |S xY w)uÖ  
        function to get data ,
        :param reporthook ä½¿ç”¨urllib.request çš„report_hook æ�¥æ±‡æŠ¥ä¸‹è½½è¿›åº¦                     å�‚è€ƒ https://docs.python.org/3/library/urllib.request.html#module-urllib.request
        :param path_to_download æ•°æ�®æ–‡ä»¶ä¸‹è½½çš„åœ°å�€ï¼Œå¦‚æžœæ²¡æœ‰æ��ä¾›ï¼Œåˆ™ä¸‹è½½åˆ°ä¸´æ—¶æ–‡ä»¶ä¸­ï¼Œå¹¶åœ¨è§£æž�ä¹‹å�Žåˆ é™¤
        :param proxies urllibæ ¼å¼�çš„ä»£ç�†æœ�åŠ¡å™¨è®¾ç½®
        :return: è§£æž�ä¹‹å�Žçš„æ•°æ�®ç»“æžœ
        r   )Ú
reporthookÚpath_to_downloadÚproxiesÚ	chunksize)r   r   r   )r   Úfetch_via_httpÚget_contentÚparseÚclose)	r   r   r   r   r   ÚargsÚkwargsÚdownload_fileÚresults	            r
   Úfetch_and_parsezBaseCralwer.fetch_and_parse   s¶   € ð �I‰I˜ÒØ/˜D×/Ñ/ð  RV¸:ÐXhÐryð  FOñ  að  Z`ñ  a‰Mà,˜D×,Ñ,ð  ~B¸
ÐUeÐqzñ  Mð  FLñ  MˆMà�$—*‘*˜]Ð<¨TÒ<°VÑ<ˆð	Ø×ÑÔ!ð ˆøð	ØØˆús   ÁA' Á'A,c                 óˆ  — |€t        j                  d¬«      }nt        |d«      } | j                  |i |¤Ž}t	        |«      }	|	j                  d|«       |	j                  dd«       t        |	«      }
t        j                  r|
j                  «       }n|
}|j                  d«      �jt        |j                  d«      j                  «       «      }d}	 |
j                  |«      }|t        |«      z  }|�	 |||«       |sn3|j                  |«       Œ?|
j                  «       }|j                  |«       |j!                  d«       |S )	NT)ÚdeleteÚwbÚRefererz
User-AgentzyMozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36zContent-Lengthr   )ÚtempfileÚNamedTemporaryFileÚopenÚget_urlr   Ú
add_headerr   ÚsixÚPY2ÚinfoÚ	getheaderÚintÚstripÚreadÚlenÚwriteÚseek)r   r   r   r   r   r   r   r   ÚurlÚrequestÚresÚresinfor	   r   ÚchunkÚcontents                   r
   r   zBaseCralwer.fetch_via_http+   s8  € ØÐ#Ü$×7Ñ7¸tÔD‰Mä Ð!1°4Ó8ˆMàˆd�l‰l˜DÐ+ FÑ+ˆä˜#“,ˆØ×Ñ˜9 cÔ*Ø×Ñ˜<ð  *fô  	gÜ�gÓˆä�7Š7Ø—h‘h“j‰GàˆGà×ÑÐ-Ó.Ð:Ü˜W×.Ñ.Ð/?Ó@×FÑFÓHÓIˆJØˆJàØŸ™ Ó+�Øœc %›jÑ(�
ØÐ)Ù˜z¨*Ô5ÙØØ×#Ñ# EÔ*ð ð —h‘h“jˆGØ×Ñ Ô(à×Ñ˜1ÔØÐr   c                 ó   — t        d«      ‚©Nzwill impl in subclass©ÚNotImplementedError)r   r   r   s      r
   r)   zBaseCralwer.get_urlQ   ó   € Ü!Ð"9Ó:Ð:r   c                 ó   — t        d«      ‚r<   r=   )r   r   r   r   r   r   r   s          r
   r   zBaseCralwer.get_contentT   r?   r   c                 ó   — t        d«      ‚r<   r=   )r   r   r   r   s       r
   r   zBaseCralwer.parseW   r?   r   )r   )NNNi È  )	Ú__name__Ú
__module__Ú__qualname__r   r!   r   r)   r   r   © r   r
   r   r      s!   „ óóó*#òL;ó;ó;r   r   )
r&   r+   Úmathr,   Úurllib2r   r   Úurllib.requestr   r   rE   r   r
   Ú<module>rI      s2   ðó Û 
Û à‡7‚7ß(Ð(ç/òG÷G;ò G;r   