U
    )3g	                     @   sZ   d dl Zd dlZd dlZddlmZ ddlmZm	Z	 e
dfddZG dd	 d	eZdS )
    N   )common)KILOBYTEMEGABYTEZint64c                    sD   | |   j  dkst fddfddt|D S )Nr   c                    s"   t tj| d|  d S )Ni  )seed)npZ
frombufferr   Zget_random_bytesview)colchunk)dtyperowsize 8/tmp/pip-unpacked-wheel-0fuzq0j6/benchmarks/streaming.pymake_column   s     
z$generate_chunks.<locals>.make_columnc                    s*   g | ]" t  fd dtD qS )c                    s    i | ]}d t | | qS )c)str).0r	   )r
   r   r   r   
<dictcomp>"   s   
 z.generate_chunks.<locals>.<listcomp>.<dictcomp>)pdZ	DataFramerange)r   )r   ncols)r
   r   
<listcomp>"   s   z#generate_chunks.<locals>.<listcomp>)itemsizeAssertionErrorr   )
total_sizenchunksr   r   r   )r   r   r   r   r   generate_chunks   s    r   c                   @   sN   e Zd ZdZde ZdZde de de gZdgZ	egZ
dd Zd	d
 ZdS )StreamReaderz>
    Benchmark in-memory streaming to a Pandas dataframe.
    @            
chunk_sizec                 C   sf   t | j| j| | jd}dd |D }|d j}t }t||}|D ]}|| qH| | _	d S )N)r   r   c                 S   s   g | ]}t j|qS r   )paZRecordBatchZfrom_pandas)r   dfr   r   r   r   :   s   z&StreamReader.setup.<locals>.<listcomp>r   )
r   r   r   schemar#   ZBufferOutputStreamZRecordBatchStreamWriterZwrite_batchgetvaluesource)selfr"   chunksZbatchesr%   ZsinkZstream_writerbatchr   r   r   setup3   s    
zStreamReader.setupc                 G   s    t | j}| }| }d S )N)r#   ZRecordBatchStreamReaderr'   Zread_allZ	to_pandas)r(   argsreadertabler$   r   r   r   time_read_to_dataframeC   s    z#StreamReader.time_read_to_dataframeN)__name__
__module____qualname____doc__r   r   r   r   Zchunk_sizesZparam_namesparamsr+   r/   r   r   r   r   r   (   s   r   )Znumpyr   Zpandasr   Zpyarrowr#    r   r   r   r   r   objectr   r   r   r   r   <module>   s   