Ë
    Þš§jQ&  ã                   ó  — d dl Z d dlmZmZ d dlmZmZmZmZ d dl	Z	erd dl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  dee   dede!e	jD                     fd„Z# G d„ d«      Z$dgZ%y)é    N)ÚIterableÚSized)ÚTYPE_CHECKINGÚAnyÚNoReturnÚUnion)Ú	DataFrameé   ©ÚCatalogé   )Ú	SparkConf)ÚSparkContext)ÚPySparkTypeError©ÚContributionsAcceptedError)ÚRuntimeConfig©ÚDataFrameReader©ÚDataStreamReader)Ú
StructType©ÚUDFRegistrationÚdataÚschemaÚreturnc           	      óî   — ddl m} g }| D ]\  }t        ||D �cg c]  }|j                  ‘Œ c}d¬«      D ��cg c]  \  }} |||j                  «      ‘Œ }}}|j                  |«       Œ^ |S c c}w c c}}w )Nr   )ÚValueF)Ústrict)Úduckdbr   ÚzipÚdataTypeÚduckdb_typeÚappend)	r   r   r   Únew_dataÚrowÚyÚxÚdtypeÚnew_rows	            úhC:\Users\Luka\AppData\Local\Temp\pip-target-o84fipzb\lib\python\duckdb/experimental/spark/sql/session.pyÚ_combine_data_and_schemar-   $   sy   € Ýà€HÛˆÜ?BÀ3Ñ]cÓHdÑ]cÐXYÈÏËÐ]cÑHdÐmrÕ?sÔtÑ?s±8°1°e‘5˜˜E×-Ñ-Õ.Ð?sˆÑtØ�‰˜Õ ð ð €Oùò IeùÓts
   ˜A,·A1c                   óò  — e Zd Zdeddfd„Zdeee   df   defd„Z	ddde
e   dz  d	e
e   dz  defd
„Z	 	 	 d'dedee   f   dee
e   z  dz  dedz  dedef
d„Zd(d„Z	 	 	 d)dededz  dededz  ddf
d„Zdededefd„Zd*d„Zdedefd„Zd(d„Zed+d„«       Zedefd„«       Zedefd„«       Zedefd „«       Zedefd!„«       Z ede!fd"„«       Z"ede#fd#„«       Z$edefd$„«       Z% G d%„ d&«      Z& e&«       Z'y),ÚSparkSessionÚcontextr   Nc                 óh   — |j                   | _        || _        t        | j                  «      | _        y ©N)Ú
connectionÚconnÚ_contextr   Ú_conf©Úselfr0   s     r,   Ú__init__zSparkSession.__init__/   s&   € Ø×&Ñ&ˆŒ	ØˆŒÜ" 4§9¡9Ó-ˆ�
ó    r   ÚPandasDataFramec                 óH  — 	 dd l }d}|rrt        |j                  «      r\dt	        j
                  «       › �}| j                  j                  ||«       t        | j                  j                  d|› d�«      | «      S dt        t           dd fd	„}t        |t        «      st        |«      } ||«       dt        dt        fd
„} ||«      }dt        dt        t           fd„} ||«      }	| j                  j                  ||	¬«      }
t        |
| «      S # t        $ r d}Y �Œw xY w)Nr   TFÚpyspark_pandas_df_zselect * from "Ú"Útuplesr   c           
      óä   — t        | «      dk  ry t        | d   «      }t        | dd  «      D ]B  \  }}t        |«      }||k(  rŒt        dd|› �d|dz   › �t        |«      t        |«      dœ¬«      ‚ y )Nr
   r   ÚLENGTH_SHOULD_BE_THE_SAMEr   )Úarg1Úarg2Úarg1_lengthÚarg2_length©Úerror_classÚmessage_parameters)ÚlenÚ	enumerater   Ústr)r?   Úexpected_lengthÚiÚitemÚactual_lengths        r,   Úverify_tuple_integrityz>SparkSession._create_dataframe.<locals>.verify_tuple_integrity@   s†   € Ü�6‹{˜aÒØÜ! &¨¡)›nˆOÜ$ V¨A¨B ZÖ0‘��4Ü # D£	�Ø" mÒ3ØÜ&Ø ;à"& q c 
Ø"& q¨1¡u g Ü'*¨?Ó';Ü'*¨=Ó'9ñ	(ôð ñ	 1r:   c           
      óÜ   — dt         dt        dt        fd„}t        | d   «      }t	        | «      D ��cg c]  \  }} ||d||z  z   «      ‘Œ }}}dj                  |«      }d|› d	�}|S c c}}w )
Nr'   Ústart_param_idxr   c                 ó�   — t        | «      }t        |«      D �cg c]
  }d||z   › �‘Œ }}ddj                  |«      z   dz   }|S c c}w )NÚ$Ú(ú, Ú))rI   ÚrangeÚjoin)r'   rR   Úparameter_countr)   Ú
parameterss        r,   Úconstruct_values_listzVSparkSession._create_dataframe.<locals>.construct_query.<locals>.construct_values_listW   sW   € Ü"% c£(�ÜAFÀÔAWÓXÑAW¸A  ! oÑ"5Ð!6Ò7ÐAW�
ÐXØ  4§9¡9¨ZÓ#8Ñ8¸3Ñ>�
Ø!Ð!ùò Ys   ™Ar   r
   rV   z'
                select * from (values z)
            )r   ÚintrK   rI   rJ   rY   )r?   r\   Úrow_sizerM   r)   Úvalues_listÚquerys          r,   Úconstruct_queryz7SparkSession._create_dataframe.<locals>.construct_queryV   s�   € ð"¬5ð "Ä3ð "Ì3ó "ô ˜6 !™9“~ˆHÜT]Ð^dÔTeÔfÑTeÉDÈAÈqÑ0°°A¸¸X¹Ñ4FÕGÐTeˆKÑfØŸ)™) KÓ0ˆKð'Ø'2 mð 4ðˆEð ˆLùó gs   ³A(c                 óL   — g }| D ]  }|j                  t        |«      «       Œ |S r2   )ÚextendÚlist)r?   r[   r'   s      r,   Úconstruct_parametersz<SparkSession._create_dataframe.<locals>.construct_parametersh   s)   € ØˆJÛ�Ø×!Ñ!¤$ s£)Õ,ð àÐr:   )Úparams)ÚpandasÚImportErrorÚ
isinstancer	   ÚuuidÚuuid1r4   ÚregisterÚsqlrd   Útupler   rK   )r8   r   rg   Ú
has_pandasÚunique_namerP   ra   r`   re   r[   Úrels              r,   Ú_create_dataframezSparkSession._create_dataframe4   s  € ð	ÛàˆJñ œ* T¨6×+;Ñ+;Ô<Ø.¬t¯z©z«|¨nÐ=ˆKØ�I‰I×Ñ˜{¨DÔ1Ü˜TŸY™YŸ]™]¨_¸[¸MÈÐ+KÓLÈdÓSÐSð	¬4´©;ð 	¸4ó 	ô$ ˜$¤Ô%Ü˜“:ˆDÙ˜tÔ$ð	¤Hð 	´ó 	ñ    Ó%ˆð	¬ð 	´d¼4±jó 	ñ *¨$Ó/ˆ
à�i‰i�m‰m˜E¨*ˆmÓ5ˆÜ˜˜dÓ#Ð#øôq ò 	Ø‹Jð	ús   ‚D ÄD!Ä D!ÚtypesÚnamesc                 ól   — | j                  |«      }|r |j                  |Ž }|r |j                  |Ž }|S r2   )rr   Ú_cast_typesÚtoDF)r8   r   rs   rt   Údfs        r,   Ú_createDataFrameFromPandasz'SparkSession._createDataFrameFromPandass   s@   € ð ×#Ñ# DÓ)ˆñ Ø�—‘ Ð'ˆBáØ�—‘˜%�ˆBØˆ	r:   r   ÚsamplingRatioÚverifySchemac                 óp  — |rt         ‚|st         ‚d }d }t        |t        «      rt        dddi¬«      ‚|r&t        |t        «      r|j                  «       \  }}n|}	 dd l}d}|r)t        |j                  «      r| j                  |||«      S d}	|s|rd}	t        d„ |D «       «      g}|rt        |t        «      rt        ||«      }| j                  |«      }
|	r)|
j                  }|j                  d	«      }t        || «      }
|r |
j                  |Ž }
|r |
j                  |Ž }
|
S # t        $ r d}Y ŒÑw xY w)
NÚSHOULD_NOT_DATAFRAMEÚarg_namer   rF   r   TFc              3   ó    K  — | ]  }d –— Œ y ­wr2   © )Ú.0Ú_s     r,   Ú	<genexpr>z/SparkSession.createDataFrame.<locals>.<genexpr>ª   s   è ø€ Ð.© 1œ$©ùs   ‚z1=0)ÚNotImplementedErrorri   r	   r   r   Úextract_types_and_namesrg   rh   ry   rn   r-   rr   ÚrelationÚfilterrv   rw   )r8   r   r   rz   r{   rs   rt   rg   ro   Úis_emptyrx   rq   s               r,   ÚcreateDataFramezSparkSession.createDataFrame€   sR  € ñ Ü%Ð%ÙÜ%Ð%ØˆØˆä�dœIÔ&Ü"Ø2Ø$.°Ð#7ôð ñ
 Ü˜&¤*Ô-Ø%×=Ñ=Ó?‘�‘uà�ð	ÛàˆJñ
 œ* T¨6×+;Ñ+;Ô<Ø×2Ñ2°4¸ÀÓFÐFð ˆÙ™àˆHÜÑ.©Ó.Ó.Ð/ˆDá”j ¬Ô4ä+¨D°&Ó9ˆDà×#Ñ# DÓ)ˆÙØ—+‘+ˆCà—*‘*˜UÓ#ˆCÜ˜3 Ó%ˆBñ Ø�—‘ Ð'ˆBáØ�—‘˜%�ˆBØˆ	øô? ò 	ØŠJð	ús   ÁD' Ä'D5Ä4D5c                 ó,   — t        | j                  «      S r2   )r/   r5   ©r8   s    r,   Ú
newSessionzSparkSession.newSession¿   s   € Ü˜DŸM™MÓ*Ð*r:   ÚstartÚendÚstepÚnumPartitionsr	   c                 ór   — |rt         ‚|€|}d}t        | j                  j                  d|||g¬«      | «      S )Nr   rX   )r[   )r   r	   r4   Útable_function)r8   r�   rŽ   r�   r�   s        r,   rX   zSparkSession.rangeÂ   sF   € ñ Ü,Ð,àˆ;ØˆCØˆEä˜Ÿ™×1Ñ1°'ÀuÈcÐSWÐFXÐ1ÓYÐ[_Ó`Ð`r:   ÚsqlQueryÚkwargsc                 ó`   — |rt         ‚| j                  j                  |«      }t        || «      S r2   )r„   r4   rm   r	   )r8   r“   r”   r†   s       r,   rm   zSparkSession.sqlÒ   s*   € ÙÜ%Ð%Ø—9‘9—=‘= Ó*ˆÜ˜ 4Ó(Ð(r:   c                 ó8   — | j                   j                  «        y r2   )r5   Ústopr‹   s    r,   r—   zSparkSession.stopØ   s   € Ø�‰×ÑÕr:   Ú	tableNamec                 óP   — | j                   j                  |«      }t        || «      S r2   )r4   Útabler	   )r8   r˜   r†   s      r,   rš   zSparkSession.tableÛ   s!   € Ø—9‘9—?‘? 9Ó-ˆÜ˜ 4Ó(Ð(r:   c                 ó   — | S r2   r€   r‹   s    r,   ÚgetActiveSessionzSparkSession.getActiveSessionß   s   € Øˆr:   c                 óX   — t        | d«      sddlm}  || «      | _        | j                  S )NÚ_catalogr   r   )ÚhasattrÚ%duckdb.experimental.spark.sql.catalogr   rž   )r8   r   s     r,   ÚcatalogzSparkSession.catalogâ   s$   € ä�t˜ZÔ(ÝEá# D›MˆDŒMØ�}‰}Ðr:   c                 ó   — | j                   S r2   )r6   r‹   s    r,   ÚconfzSparkSession.confê   s   € à�z‰zÐr:   c                 ó   — t        | «      S r2   r   r‹   s    r,   ÚreadzSparkSession.readî   ó   € ä˜tÓ$Ð$r:   c                 ó   — t        | «      S r2   r   r‹   s    r,   Ú
readStreamzSparkSession.readStreamò   s   € ä Ó%Ð%r:   c                 ó   — | j                   S r2   )r5   r‹   s    r,   ÚsparkContextzSparkSession.sparkContextö   s   € à�}‰}Ðr:   c                 ó   — t         ‚r2   r   r‹   s    r,   ÚstreamszSparkSession.streamsú   s   € ä(Ð(r:   c                 ó   — t        | «      S r2   r   r‹   s    r,   ÚudfzSparkSession.udfþ   r¦   r:   c                  ó   — y)Nz1.0.0r€   r‹   s    r,   ÚversionzSparkSession.version  s   € àr:   c            	       ó†   — e Zd Zdd„Zdedd fd„Zdedd fd„Zdedd fd„Zdd	„Z	 	 	 dd
edz  de	dz  de
dz  dd fd„Zdd„Zy)úSparkSession.Builderr   Nc                  ó   — y r2   r€   r‹   s    r,   r9   zSparkSession.Builder.__init__  s   € Ør:   Únamec                 ó   — | S r2   r€   ©r8   r´   s     r,   ÚmasterzSparkSession.Builder.master
  ó   € àˆKr:   c                 ó   — | S r2   r€   r¶   s     r,   ÚappNamezSparkSession.Builder.appName  r¸   r:   Úurlc                 ó   — | S r2   r€   )r8   r»   s     r,   ÚremotezSparkSession.Builder.remote  r¸   r:   c                 ó.   — t        d«      }t        |«      S )NÚ__ignored__)r   r/   r7   s     r,   ÚgetOrCreatez SparkSession.Builder.getOrCreate  s   € Ü" =Ó1ˆGÜ Ó(Ð(r:   ÚkeyÚvaluer£   c                 ó   — | S r2   r€   )r8   rÁ   rÂ   r£   s       r,   ÚconfigzSparkSession.Builder.config  s	   € ð ˆKr:   c                 ó   — | S r2   r€   r‹   s    r,   ÚenableHiveSupportz&SparkSession.Builder.enableHiveSupport"  r¸   r:   ©r   N©r   r/   )NNN)r   r²   )Ú__name__Ú
__module__Ú__qualname__r9   rK   r·   rº   r½   rÀ   r   r   rÄ   rÆ   r€   r:   r,   ÚBuilderr²     s�   „ ó	ð	˜sð 	Ð'=ó 	ð	 ð 	Ð(>ó 	ð	˜cð 	Ð&<ó 	ó	)ð #Ø $Ø%)ñ		à�t‘ð	ð ˜‘:ð	ð ˜dÑ"ð		ð
 $ó	ô	r:   rÌ   )NNTrÈ   )Nr
   NrÇ   )r   r   )(rÉ   rÊ   rË   r   r9   r   r   r   r	   rr   rd   rK   ry   r   ÚfloatÚboolr‰   rŒ   r]   rX   rm   r—   rš   rœ   Úpropertyr¡   r   r£   r   r¥   r   r¨   rª   r   r¬   r   r®   r°   rÌ   Úbuilderr€   r:   r,   r/   r/   .   sC  „ ð. ð .°ó .ð
=$ e¨H°S©MÐ;LÐ,LÑ&Mð =$ÐR[ó =$ð~Ø%ðØ.2°3©i¸$Ñ.>ðØGKÈCÁyÐSWÑGWðà	óð  15Ø&*Ø!ñ=àÐ% x°¡}Ð4Ñ5ð=ð ˜T #™YÑ&¨Ñ-ð=ð ˜t‘|ð	=ð
 ð=ð 
ó=ó~+ð ØØ$(ñaàðað �4‰Zðað ð	að
 ˜T‘zðað 
óað )˜Cð )¨3ð )°9ó )óð)˜sð ) yó )óð òó ðð ð�mò ó ðð ð%�oò %ó ð%ð ð&Ð,ò &ó ð&ð ð˜lò ó ðð ð)˜ò )ó ð)ð ð%�_ò %ó ð%ð ð˜ò ó ð÷ñ ñ@ ‹i�Gr:   r/   )&rj   Úcollections.abcr   r   Útypingr   r   r   r   r!   Úpandas.core.framer	   r;   r¡   r   r£   r   r0   r   Úerrorsr   Ú	exceptionr   r   Ú	dataframeÚ
readwriterr   Ú	streamingr   rs   r   r®   r   rd   r   r-   r/   Ú__all__r€   r:   r,   Ú<module>rÚ      s{   ðÛ ß +ß 6Ó 6ã áÝ>å õ Ý "Ý %Ý 2Ý Ý  Ý 'Ý 'Ý Ý  ð 8¨C¡=ð ¸*ð ÈÈfÏlÉlÑI[ó ÷xñ xðv Ð
�r:   