*jWddZddlmZddlZddlmZmZddlZddl m Z m Z m Z ddl mZddlmZmZddlmZmZdd lmZdd lmZer2dd lmZmZmZdd lmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(ddl)m*Z*ddl+m,Z, d dZ- d!dZ. d"dZ/d#dZ0 d$dZ1 d%dZ2 d& d'dZ3 d( d)dZ4d*d+dZ5d,dZ6 d- d.dZ7 d- d/dZ8 d0dZ9 d- d1dZ: d2 d3dZ; d4dZ > FF9 &"@"@ FM *&,&C&C&EF&Es&EF! )$  N  4+#   NGs C%cdd}dd}|Dcgc] }t|}}t|}|s4tt||dD]\}\} } || | \||<||< ||} t j |d| d} | |dj dd d z} td| D]3}||dk(rt jd} n| ||z} | ||| zz } 5|r|dd k(}|d| D] } || d k(z} d | |<| t|k(r | St| | \}}|g|| d }t|g|| d }cc}w)a{ For the particular label_list, gets the offsets into the hypothetical list representing the totally ordered cartesian product of all possible label combinations, *as long as* this space fits within int64 bounds; otherwise, though group indices identify unique combinations of labels, they cannot be deconstructed. - If `sort`, rank of returned ids preserve lexical ranks of labels. i.e. returned id's can be used to do lexical sort on labels; - If `xnull` nulls (-1 labels) are passed through. Parameters ---------- labels : sequence of arrays Integers identifying levels at each location shape : tuple[int, ...] Number of unique levels at each location sort : bool If the ranks of returned ids should match lexical ranks of labels xnull : bool If true nulls are excluded. i.e. -1 values in the labels are passed through. Returns ------- An array of type int64 where two elements are equal if their corresponding labels are equal at all location. Notes ----- The length of `labels` and `shape` must be identical. cd}t|D]*\}}|t|z}|tjkr(|cSt |SN) enumerateintri8maxlen)shapeaccimuls r<_int64_cut_offz'get_group_index.._int64_cut_offsC&FAs 3s8OC?'5zcD|dk(jr |dz|dzfS||fS)NrAr.)labsizes r< maybe_liftz#get_group_index..maybe_lifts-(+byoo&7a"Hc4[HrKTstrictrAi8dtyperFsubokcopyrMsortN)returnrC)rPrCr\ztuple[np.ndarray, int]) r listrBzipr+prodastyperangeint64rEcompress_group_index)labelsrFr[xnullrJrQxlshaperHrOrPnlevstrideoutmaskcomp_idsobs_idss r<get_group_indexrnysFI (. .v!l1ovF . %[F 'FE$(GHNA{T#-c4#8 F1IvayI f%$t4vay''E'FFq$AayA~!6!9$ 6!9v% %C   !9?Da~r !&CI 3v;   J 14@'+VDE]+g,//; /sD=c:t||dd}t|dS)a Group_index is offsets into cartesian product of all possible labels. This space can be huge, so this function compresses it, by computing offsets (comp_ids) into the list of unique labels (obs_group_ids). Parameters ---------- labels : list of label arrays sizes : tuple[int] of size of the levels Returns ------- np.ndarray[np.intp] comp_ids np.ndarray[np.int64] obs_group_ids TFr[rerZ)rnrc)rdsizesidss r<get_compressed_idsrss!( &%d% @C $ //rKcVd}|D]}|t|z}|tjk\Sr@)rCrrD)rFthe_prodrfs r<is_int64_overflow_possiblervs0H CF syy  rKcJt|r tdg}d}tjd}|}t t t |D]K}||z |||zz|z}tj||dkd|j|||z}|||z}M|dddS)Nz,cannot deconstruct factorized group indices!rArrM) rv ValueErrorr+arrayreversedrarEputmaskappend) comp_labelsrF label_listfactoryrfrHrds r<_decons_group_indexrs"%(GHHJ F  AA eCJ' (a%FU1X-.&8 6;?B/&! VO%( ) dd rKc|s_tjd|Dtj}tj|tj|z}t |}t |sDt ||}|sjs|St||dD cgc] \}} || z  c} }St|} |D cgc]'} | | jtjdd)c} Scc} }wcc} w)a6 Reconstruct labels from observed group ids. Parameters ---------- comp_ids : np.ndarray[np.intp] obs_ids: np.ndarray[np.intp] shape : tuple[int] labels : Sequence[np.ndarray[np.signedinteger]] xnull : bool If nulls are excluded; i.e. -1 labels are passed through. c3BK|]}|dk(jyw)rMNrN).0as r< z'decons_obs_group_ids..s??&$!!&#d)4 !!$D 2&! Ft<5 a % Q'E#&u: a1Aa.CIIECNN#A{ & TXXZHHT1e,EHHT5!e)a-8E e'=* ::f rKc |t||}t||||d|St|tr|j |St|t s t |}n td|tjt|}t|tjs|j |||Stjt|}||}||}tj|d} |s|ddd}|ddd}||j |} |s| ddd} |d k(r"tj| | g} t#| S|d k(r"tj| | g} t#| St!d |) al Intended to be a drop-in replacement for np.argsort which handles NaNs. Adds ascending, na_position, and key parameters. (GH #6399, #5231, #27237) Parameters ---------- items : np.ndarray, ExtensionArray, Index, or Series kind : {'quicksort', 'mergesort', 'heapsort', 'stable'}, default 'quicksort' ascending : bool, default True na_position : {'first', 'last'}, default 'last' key : Optional[Callable], default None mask : Optional[np.ndarray[bool]], default None Passed when called by ExtensionArray.argsort. Returns ------- np.ndarray[np.intp] N)r'r"r$r8rk)r"zDnargsort does not support MultiIndex. Use index.sort_values instead.)r"r'r$rrMr'rrr)r(r4r0r argsortr r TypeErrorr+rrrarangerEnonzero concatenaterxr ) itemsr'r"r$r8rkidxnon_nans non_nan_idxnan_idxr:s r<r4r4ts< !%- #   %'}}y}11 } -e$ R   |zz$u+& eRZZ (}}#  ))CJ CdU|Hte*Kjjq!G DbD>!$B$' (***56G $B$-f..'7!34 w ''  ..'7!34 w ''0 >??rKc |dvsJ|dk(rtjntj}tjt |}|j }|j dkDr|jri|dk(rt||d}n"t|j|jd}tj|Dcgc]\}}t|||c}}S|||St|||Scc}}w)a Implementation of np.argmin/argmax but for ExtensionArray and which handles missing values. Parameters ---------- values : ExtensionArray method : {"argmax", "argmin"} axis : int, default 0 Returns ------- int >argmaxargminrrATrR)axis) r+rrrr_values_for_argsortndimr.r^Try _nanargminmax) valuesmethodrfuncrk arr_valueszippedvms r< nargminmaxrs ) )) )(*299 D ::d6l #D++-J 88:qyZd;Z\\466$?886J641a]1a66JK KJT** T4 00KsC9 cttj|jd}||}||}|||S)z# See nanargminmax.__doc__. r)r+rrF)rrkrrrrs r<rrs? ))FLLO $Cte}Hte*K tH~ &&rKc|;t|ttfr|g}n|}|Dchc]}|j|}}nt |j }t |j Dcgc]2}||vrt |j||n|j|4}}t|j|Scc}wcc}w)ad Returns a new MultiIndex in which key has been applied to all levels specified in level (or all levels if level is None). Used for key sorting for MultiIndex. Parameters ---------- index : MultiIndex Index to which to apply the key function on the specified levels. key : Callable Function that takes an Index and returns an Index of the same shape. This key is applied to each level separately. The name of the level can be used to distinguish different levels for application. level : list-like, int or str, default None Level or list of levels to apply the key function to. If None, key function is applied to all levels. Other levels are left unchanged. Returns ------- labels : MultiIndex Resulting MultiIndex with modified levels. ) r0strrC_get_level_numberranlevelsr(_get_level_valuestype from_arrays)indexr8r7 level_iterr; sort_levelsmappeds r<_ensure_key_mapped_multiindexrs:  ec3Z (JJLV#WJSE$;$;C$@J #WEMM* 5==)  *E # e55e>Cctd|D}t||dd}tj|dk(riSt |r!|j xr|j dzntj|d}t||}|Dcgc]}|j|}}|j|}tj||||Scc}w)zJ Returns ------- dict: Labels mapped to indexers. c32K|]}t|ywN)rE)rrfs r<rz#get_indexer_dict..`s'$Q#a&$sTrprMrArTrU) rrnr+r,rvrPrr_get_group_index_sortertaker indices_fast)r~rrF group_indexngroupssorterrO sorted_labelss r<get_indexer_dictrWs '$' 'E!*e$dKK vvkR   &e ,    0{0A 5 WWU$ ' $K 9F1;<#SXXf%M<""6*K   FK} EE=sCc2|d|jz}t|}d}d}|dkDxr!|||zz|tj|zk}|r-t j t ||\}}t |S|jd}t |S)a  algos.groupsort_indexer implements `counting sort` and it is at least O(ngroups), where ngroups = prod(shape) shape = map(len, keys) that is, linear in the number of combinations (cartesian product) of unique values of groupby keys. This can be huge when doing multi-key groupby. np.argsort(kind='mergesort') is O(count x log(count)) where count is the length of the data-frame; Both algorithms are `stable` sort and that is necessary for correctness of groupby operations. e.g. consider: df.groupby(key)[col].transform('first') Parameters ---------- group_index : np.ndarray[np.intp] signed integer dtype ngroups : int or None, default None Returns ------- np.ndarray[np.intp] rAgg?r mergesortr)rrEr+logrgroupsort_indexerr r)rrcountalphabeta do_groupsortrr9s r<rrxs4koo''  E E D19U54'>#9ebffUm>S"TL++  ,   v &&$$+$6 v &&rKct|r_tj|dd|ddk\rAtj|dddkD|dd|ddk7g}|j }|dz}||}n^t|}t j |}t|}|j|\}}|rt|dkDrt||\}}t|t|fS)z Group_index is offsets into cartesian product of all possible labels. This space can be huge, so this function compresses it, by computing offsets (comp_ids) into the list of unique labels (obs_group_ids). rANrMr) rEr+r,rcumsumrInt64HashTabler get_labels_groupby_reorder_by_uniques)rr[ unique_maskrl obs_group_ids size_hinttables r<rcrcs ;BFF;qr?k#26F#FGnn !_r !;qr?k#26F#F G %%'A #K0  $ ((3";/ #(":":;"G- C &*&9-&R #M8  !< #> >>rKc^|j}tjt|tj}|j |tj t||dk}|j|}tj||d|j|}||fS)z Parameters ---------- uniques : np.ndarray[np.int64] labels : np.ndarray[np.intp] Returns ------- np.ndarray[np.int64] np.ndarray[np.intp] rUrrM) rr+emptyrErputrrr{)uniquesrdrreverse_indexerrks r<rrs__ Fhhs6{"'':O #f+ 67 A:D ! !& )FJJvtR ll6"G F?rK)r6rr7zLevel | list[Level] | Noner"zlist[bool] | boolr'rr$rr#r5r8rr\znpt.NDArray[np.intp] | None)rFrr[r5rer5r\npt.NDArray[np.int64])rqrr\z2tuple[npt.NDArray[np.intp], npt.NDArray[np.int64]])rFrr\r5)r}npt.NDArray[np.intp]rFrr\list[npt.NDArray[np.intp]]) rlrrmrrFrrdz'Sequence[npt.NDArray[np.signedinteger]]rer5r\r)NrNF) rz$Sequence[ArrayLike | Index | Series]r$rr8Callable | Noner&r5r\r) quicksortTrNN)rArrayLike | Index | Seriesr'rr"r5r$rr8rrkznpt.NDArray[np.bool_] | Noner\r)r)rrrrrr)rz np.ndarrayrkznpt.NDArray[np.bool_]r\rCr)rrr8rr\r)rrr8rr\r)r~zlist[np.ndarray]rz list[Index]r\z$dict[Hashable, npt.NDArray[np.intp]])rrrz int | Noner\r)T)rrr[r5r\z3tuple[npt.NDArray[np.int64], npt.NDArray[np.int64]])rrrdrr\z2tuple[npt.NDArray[np.int64], npt.NDArray[np.intp]])=__doc__ __future__rrtypingrrnumpyr+ pandas._libsrrrpandas._libs.hashtabler pandas.core.dtypes.commonr r pandas.core.dtypes.genericr r pandas.core.dtypes.missingrpandas.core.constructionrcollections.abcrrrpandas._typingrrrrrrrrpandasrrrrpandas.core.indexes.baserr=rnrsrvrrr3r4rrrr(rrrcrrKr<rs/"  8,2   2.< < %<!<  <  <  < <!<~WW $W-1WWt00700!%.3,"T""T !"T "T 4 "T  "T  "TN  C .CC  C  C  CP!)- Q( %Q( Q(Q( Q(  Q( ' Q(Q(h1B'-10+ 0+$0+0+hFJ- &--<--`F F(3F)FD>B('%('0:('('X6:?&?.2?8?> ",@7rK