o
    EhQ                     @  s.  d Z ddlmZ ddlmZ ddlZddlZddlZddlmZ ddlZddl	Z	ddl
Z
ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddlZddlmZmZ dd	lmZ dd
lmZ ddlm  m Z! ddl"Z#ddl"m$Z$m%Z%m&Z&m'Z' ddl(m)Z* ddl+m,Z, ddl-m	  m.Z/ ddl0m1Z1m2Z2 dd Z3ej45ddde6de7de8dedg de7d9dde7d9ddddd e:dj;ddde:dj;ddde:dj;ddddddddf gdd Z<dd  Z=d!d" Z>d#d$ Z?dsd(d)Z@ej45d*ejAe>d+d,ejAe#jBd-d,ejAeje#jBejCd.d/d,ejAeje#jBd0d.d1d,ejAeje#jBd2d.d3d,gej45d4e#jBe>gej45d5e@e, d6d7 ZDd8d9 ZEd:d; ZFejGd<d= ZHG d>d? d?ZIG d@dA dAZJej45dBdCgdDe%g dEeKdFfdGe#Le%g dEeKdFe%g dHeKdFgeg dIeg dIgfgdJdK ZMdLdM ZNej45dNg dOdPdQ ZOdRdS ZPG dTdU dUejQZRdVdW ZSdXdY ZTdZd[ ZUd\d] ZVej45d^d_d`d^edadbdbdcgddde ZWdfdg ZXdhdi ZYe!jZdjdk Z[ej45dlej\ejCgdmdn Z]dodp Z^dqdr Z_dS )ta<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )annotations)arrayN)partial)Path)Any)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)	DataFrameIndexSeriesperiod_range)create_pickle_data)DayMonthEndc                 C  s   t |trt||  d S |drt| | d S |dkr4|tju r,| tju s*J d S | |ks2J d S ttd| dtj	}|| | d S )NZsp_Z	timestampZassert_Z_equal)

isinstancer   tmassert_index_equal
startswithZassert_equalpdZNaTgetattrZassert_almost_equal)resultexpectedtypZ
comparator r   ;lib/python3.10/site-packages/pandas/tests/io/test_pickle.pycompare_element>   s   


r   datas   123s   123456I         B)r#   r"   r"      )r#      C)ZorderFc                 C  s   t | }t| d}||ksJ t| ttfr || u sJ d S t|trA|jdks,J |jdks3J |js8J |j	|j
fksCJ d S d S )NAr!   r$   )r
   
memoryviewtobytesr   bytes	bytearrayndimformat
contiguousshapenbytes)r   r   r   r   r   r   test_flatten_bufferT   s   

r3   c           
   	   C  s  t  std ttjdD ]}| |}t|}|	 D ]\}}|	 D ]\}}|| | }|dkr|dkrt
|| |jj|jjksIJ |jjjrPJ t
|dk|dk |jj}|td tdksjJ |tjdd }	t|	tjszJ |	tjddd	ksJ |tjdd
 }	t|	tjsJ |	tjdddksJ q'|dkr|dkrt
|| t|jtsJ |jt ksJ |jdksJ t
|d|d q'|dkr|dv rt
|| q'|dkr|dv rt
|| q't||| q'qqd S )Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*Zseriestsr   r!   r"   )hours)daysr5   )nanoseconds)r6   r7   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   __file__parentZglobr   read_pickleitemsr   assert_series_equalr8   freq	normalizer   Z	Timedeltar   r   r   freqstrshiftassert_frame_equalr   )
datapathlegacy_pickler   r   ZdvZdtr   r   rC   resr   r   r   test_picklesq   sF   

rK   c                 C  @   t |d}tj| |dd W d    d S 1 sw   Y  d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler      "rZ   c                 C  sB   t | d}|d t|W  d    S 1 sw   Y  d S )Nrbr   )rS   seekrT   load)rX   rY   r   r   r   python_unpickler   s   
$r_   dictreturnlist[tuple[str, Any]]c                 C  s   dd |   D S )zFlatten create_pickle_datac                 S  s&   g | ]\}}|  D ]}||fq
qS r   )values).0r   ZexamplesZexampler   r   r   
<listcomp>   s    zflatten.<locals>.<listcomp>)rA   )r   r   r   r   flatten   s   rf   pickle_writerZpython)idZpandas_proto_defaultrP   Zpandas_proto_highestr&   Zpandas_proto_4   Zpandas_proto_5writerztyp, expectedc              	   C  s   t  i}||| t|}t|||  t|}t|||  t|dd}||| |d W d    n1 s:w   Y  t|dd}t|}|d W d    n1 sZw   Y  t|||  W d    d S 1 spw   Y  d S )NrN   moder   r\   )r   ensure_cleanr   r@   r   r_   rS   r]   )r   r   rg   rj   rX   r   Zhandler   r   r   test_round_trip_current   s    




"rn   c                  C  `   t dtdd ttdtdtdd tdD tdd	} t	| j
tj}t| | d S )
N皙?x      r&   ABCDZdtypec                 S     g | ]}d | qS zi-r   rd   ir   r   r   re          z,test_pickle_path_pathlib.<locals>.<listcomp>rs   columnsr8   )r   nparangereshaper   listobjectranger   round_trip_pathlib	to_pickler   r@   rG   dfr   r   r   r   test_pickle_path_pathlib      r   c                  C  ro   )
Nrp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re      rz   z.test_pickle_path_localpath.<locals>.<listcomp>rs   r{   )r   r}   r~   r   r   r   r   r   r   Zround_trip_localpathr   r   r@   rG   r   r   r   r   test_pickle_path_localpath   r   r   c                   C  s   dt   dS )N__z	__.pickle)uuidZuuid4r   r   r   r   get_random_path   s   r   c                   @  sT   e Zd ZejZdd Zdd Zej	
dg ddd Zd	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C  s  |d u rt || d S |dkrt|d}n|dkr"t|d}n|dkrJtj|dtjd}|	|t
j| W d    n1 sDw   Y  nb|dkrt|d/}tj|dd}||t
j|}||| W d    n1 svw   Y  W d    n1 sw   Y  n!|d	krt |d}n|d
krtd|d}n	d| }t||dvrt|d*}| |	|  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S d S )Ngzipwbz2zipcompressiontarr\   rk   xzZzstdZ	zstandardrN   zUnrecognized compression type: )r   r   )shutilcopyfiler   rS   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosrX   basenametarfileZ
gettarinfoZaddfiler   r	   
ValueErrorread)	selfZsrc_pathZ	dest_pathr   frY   r   tarinfomsgr   r   r   compress_file   sH   
"zTestCompression.compress_filec                 C  sL  |}|d }|d }t |}t |o}tdtdd ttdtdtdd	 t	d
D tdd}|j
||d t j||d$}	t|d}
|
|	  W d    n1 s\w   Y  W d    n1 skw   Y  tj|d d}t || W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )N.compressed.rawrp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re   '  rz   z7TestCompression.test_write_explicit.<locals>.<listcomp>rs   r{   r   rN   )r   rm   r   r}   r~   r   r   r   r   r   r   decompress_filerS   r   r   r   r@   rG   )r   r   r   basepath1path2p1p2r   r   rY   df2r   r   r   test_write_explicit  s&   Pz#TestCompression.test_write_explicitr   ) NoneZbadZ7zc              	   C  s   t jtddM t|/}tdtdd t	t
dtdt	dd	 td
D tdd}|j||d W d    n1 s?w   Y  W d    d S W d    d S 1 sWw   Y  d S )NzUnrecognized compression typematchrp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re   >  rz   z;TestCompression.test_write_explicit_bad.<locals>.<listcomp>rs   r{   r   )r<   Zraisesr   r   rm   r   r}   r~   r   r   r   r   r   r   )r   r   r   rX   r   r   r   r   test_write_explicit_bad7  s   "z'TestCompression.test_write_explicit_badc                 C  sX  |}|| }|d }| j | }t|}t|m}tdtdd t	t
dtdt	dd td	D tdd
}	|	| tj||d$}
t|d}||
  W d    n1 sbw   Y  W d    n1 sqw   Y  tj|d d}t|	| W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   rp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re   L  rz   z4TestCompression.test_write_infer.<locals>.<listcomp>rs   r{   r   rN   )_extension_to_compressiongetlowerr   rm   r   r}   r~   r   r   r   r   r   r   r   rS   r   r   r   r@   rG   )r   compression_extr   r   r   r   r   r   r   r   r   rY   r   r   r   r   test_write_inferB  s(   
Pz TestCompression.test_write_inferc           
   	   C  s   |}|d }|d }t |b}t |D}tdtdd ttdtdtdd	 t	d
D tdd}|j
|d d | j|||d tj||d}	t ||	 W d    n1 s\w   Y  W d    d S W d    d S 1 stw   Y  d S )Nr   r   rp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re   e  rz   z6TestCompression.test_read_explicit.<locals>.<listcomp>rs   r{   r   )r   rm   r   r}   r~   r   r   r   r   r   r   r   r   r@   rG   )
r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_explicit\  s   Pz"TestCompression.test_read_explicitc              	   C  s  |}|d }|| }| j | }t|`}t|B}tdtdd t	t
dtdt	dd td	D tdd
}	|	j|d d | j|||d t|}
t|	|
 W d    n1 sbw   Y  W d    d S W d    d S 1 szw   Y  d S )Nr   rp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re   |  rz   z3TestCompression.test_read_infer.<locals>.<listcomp>rs   r{   r   )r   r   r   r   rm   r   r}   r~   r   r   r   r   r   r   r   r   r@   rG   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_inferr  s   
PzTestCompression.test_read_inferN)__name__
__module____qualname__icomZextension_to_compressionr   r   r   r<   markparametrizer   r   r   r   r   r   r   r   r      s    

r   c                   @  s&   e Zd Zejdg ddd ZdS )TestProtocolrQ   )rO   r   r!   r"   c                 C  s   t |;}tdtdd ttdtdtdd t	dD tdd	}|j
||d
 t|}t || W d    d S 1 sCw   Y  d S )Nrp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re     rz   z*TestProtocol.test_read.<locals>.<listcomp>rs   r{   rP   )r   rm   r   r}   r~   r   r   r   r   r   r   r   r@   rG   )r   rQ   r   rX   r   r   r   r   r   	test_read  s   
"zTestProtocol.test_readN)r   r   r   r<   r   r   r   r   r   r   r   r     s    r   pickle_fileexcolsztest_py27.pkl)abcru   ztest_mi_py27.pkl)r)   r$   r'   )r   r!   r"   c                 C  s*   | ddd|}t |}t|j| d S )Nior   rT   )r   r@   r   r   r|   )rH   r   r   rX   r   r   r   r   test_unicode_decode_error  s   
r   c               	   C  s   t  c} tdtdd ttdtdtdd t	dD tdd	}t
| d
}|| W d    n1 s:w   Y  t
| d}t|}W d    n1 sTw   Y  t || W d    d S 1 sjw   Y  d S )Nrp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re     rz   z0test_pickle_buffer_roundtrip.<locals>.<listcomp>rs   r{   rN   r\   )r   rm   r   r}   r~   r   r   r   r   r   rS   r   r   r@   rG   )rX   r   rY   r   r   r   r   test_pickle_buffer_roundtrip  s   
"r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                   s   dd }G dd d t  F fdd}tdtdd	 ttd
tdtdd t	dD tdd}|| | 
d| t|}t || W d    d S 1 sXw   Y  d S )Nc                 S  rL   rM   rR   rV   r   r   r   rZ     r[   z3test_pickle_generalurl_read.<locals>.python_picklerc                   @  s6   e Zd ZdddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsera   r   c                 S  s0   t |d| _d|v rddi| _d S ddi| _d S )Nr\   r   zContent-Encodingr   )rS   fileZheaders)r   rX   r   r   r   __init__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S  s   | S Nr   r   r   r   r   	__enter__     z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W  s   |    d S r   )close)r   argsr   r   r   __exit__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S  
   | j  S r   )r   r   r   r   r   r   r        
z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S  r   r   )r   r   r   r   r   r   r     r   z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeNra   r   )r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s    
r   c                    s    S r   r   )r   kwargsr   rX   r   r   mock_urlopen_read  s   z6test_pickle_generalurl_read.<locals>.mock_urlopen_readrp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re     rz   z/test_pickle_generalurl_read.<locals>.<listcomp>rs   r{   zurllib.request.urlopen)r   rm   r   r}   r~   r   r   r   r   r   setattrr   r@   rG   )Zmonkeypatchr   rZ   r   r   r   r   r   r   test_pickle_generalurl_read  s   


"r   c                  C  s   t d t ; d} tdtdd tt	dt
dtdd	 td
D t
dd}||  t| }t|| W d    d S 1 sGw   Y  d S )NZfsspeczmemory://mockfilerp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re     rz   z0test_pickle_fsspec_roundtrip.<locals>.<listcomp>rs   r{   )r<   Zimportorskipr   rm   r   r}   r~   r   r   r   r   r   r   r   r@   rG   )r   r   r   r   r   r   test_pickle_fsspec_roundtrip  s   



"r   c                   @  s   e Zd ZdddZdS )MyTzra   r   c                 C  s   d S r   r   r   r   r   r   r   
  r   zMyTz.__init__Nr   )r   r   r   r   r   r   r   r   r   	  s    r   c                  C  sD   t tdt f} t| }t|d | d  t|d ts J d S )Nru   r   r!   )r   r   r   r   round_trip_picklerB   r   )r   r   r   r   r   test_read_pickle_with_subclass  s   
r   c                 C  s   t dtdd ttdtdtdd tdD tdd	}t	 }|j
|| d
 t| }W d   n1 s<w   Y  t }|j
|| d
 |d | |ks]| dv s]J tj|| d
}|d t|| dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    rp   rq   rr   rt   ru   c                 S  rv   rw   r   rx   r   r   r   re      rz   z9test_pickle_binary_object_compression.<locals>.<listcomp>rs   r{   r   Nr   )r   r   r   )r   r}   r~   r   r   r   r   r   r   rm   r   r   Z
read_bytesr   BytesIOr]   getvaluer   r@   rG   )r   r   rX   Z	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    


r   c                 C  s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S  s   t | }t | | d S r   )r   r   rG   )r;   	unpickledr   r   r   _test_roundtrip=     
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr;   r   r   r   r   +test_pickle_dataframe_with_multilevel_index6  s   
r   c                  C  sH   t dddd} ttjdt| | }t|}|j	j
dks"J d S )Nz1/1/2011z1/1/2012r9   )rC   r"   )r   r   r}   ZrandomZdefault_rngZstandard_normallenr   r   r8   rE   )Zprngr4   Znew_tsr   r   r   "test_pickle_timeseries_periodindexG  s   
r   namei	  g     H@i     )r!   r"   c                 C  s0   t ttjdtjd| d}|j| ksJ d S )N
   ru   )r   )r   r   r   r}   r~   Zfloat64r   )r   r   r   r   r   test_pickle_preserve_nameO  s   r   c                 C     t | }t ||  d S r   r   r   rB   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimesW  r   r   c                 C  r   r   r   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_strings\  r   r   c                  C  sf   t tddjdg } t| }|jjd jdksJ |jjd j	dks(J t
|dg |  d S )Nabccategoryr   r!   )r!   T)r   r   ZastypeZilocr   r   Z_mgrZblocksr.   r1   rB   )ZserrJ   r   r   r    test_pickle_preserves_block_ndima  s
   
r   rQ   c                 C  s>   t td}tt|j| |dttj|d}t|| d S )Ni )rQ   r   r   )	r   r   r   r   r   r   r   r@   rG   )rQ   r   r   r   r   r   r   %test_pickle_big_dataframe_compressionn  s   r   c                 C  sf   | t tjdddd}t|d}t|}W d    n1 s w   Y  tg g d}t|| d S )Nr   rI   z1.2.4zempty_frame_v1_2_4-GH#42345.pklr\   )r8   r|   )	r   r>   r?   rS   rT   r^   r   r   rG   )rH   rX   fdr   r   r   r   r   #test_pickle_frame_v124_unpickle_130y  s   r   c                  C  s\   t dg di} d}tjt|d t }| |d W d    d S 1 s'w   Y  d S )Nr   r    zpStarting with pandas version 3.0 all arguments of to_pickle except for the argument 'path' will be keyword-only.r   Zinfer)r   r   Zassert_produces_warningFutureWarningr   r   r   )r   r   r   r   r   r    test_pickle_pos_args_deprecation  s   "r   )r   r`   ra   rb   )`__doc__Z
__future__r   r   r   Zdatetime	functoolsr   r   r   r   Zpathlibr   rT   r   r   typingr   r   r   Znumpyr}   r<   Zpandas.compatr   r   Zpandas.compat._optionalr	   Zpandas.compat.compressorsr
   Zpandas.util._test_decoratorsutilZ_test_decoratorsZtdZpandasr   r   r   r   r   Zpandas._testingZ_testingr   Z-pandas.tests.io.generate_legacy_storage_filesr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   r   r   r   r-   r*   ZPickleBuffercastr~   r   r3   rK   rZ   r_   rf   Zparamr   ZHIGHEST_PROTOCOLrn   r   r   Zfixturer   r   r   r   Z
MultiIndexr   r   r   r   Ztzinfor   r   r   r   r   r   r   r   Zskip_array_manager_invalid_testr   ZDEFAULT_PROTOCOLr   r   r   r   r   r   r   <module>   s    	$
2
	

 

)	



