uj| dZddlmZddlZddlZddlZddlZddlZddlm Z ddl Z ddl Z ddl Z ddlmZddlmZddlmZmZmZe rddlmZddlZddlmZdd lmZ dd l!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(ddd Z)Gd dZ*GddZ+GddZ,e jZj\GddZ/e jZjade jbddde jbddde jbdddgdZ2d Z3d!Z4e jZjked"#d$Z6e jZjad%gd&d'Z7d(Z8d)Z9Gd*d+Z:d,Z;d-Zd0Z?d1Z@e jZjad2d3ifd4ifd5d6d7ifd8d7dd9fgd:ZAe jZjad;gd<d=ZBe jZjked"#d>ZCd?ZDe jZjked@#dAZEdBZFdCZGe jZjdDdEZIdFZJe jZjadGdHde jgdIZLe jZjadJidKdLgidKdMdLgiidNdOigdPZMe jZjked"#dQZNdRZOdSZPdTZQdUZRdVZSdWZTddXZUddYZVddZZWe jZjked"#d[ZXe jZjked"#d\ZYe jZjked"#e jZjadJd]d^d_iid`d^eZiigdaZ[e jZjked"#dbZ\dcZ]ddZ^deZ_dfZ`dgZadhZbddiZcddjZde jZjadkdlgdldmgge jZjkedn#doZee jZjkedp#dqZfe jZjkedr#dsZgdtZhe jZjkedr#duZie jZjkedv#dwZje jZjkedx#dyZke jZjadJdzd{id|d}igd~Zle jZjadddge jZjked#dZme jZjked#dZne jZjked#dZoddd ddZpdddZqe jZjked@#dZre jZjked@#dZse jZjked@#dZtdZue jZjadddgdZve jZjked@#e jZjadgddZwe jZjked@#e jZjadddge jZjadddgdZxe jZjked@#dZye jZjked@#dZzdddZ{e jZjadd7dgdZ|e jZjadd7dgdZ}e jZjadd7dgdZ~e jZjadd7dgdZy)af Tests for the parallel read_csv implementation (C engine, large local files). The parallel path is enabled automatically when: - reading from a local uncompressed file path - using the C engine (the default) - not in iterator / chunked mode - file size >= _PARALLEL_READ_MIN_BYTES We test correctness (parallel == serial) rather than performance. ) annotationsN) TYPE_CHECKING)parsers)WASM)EmptyDataError ParserError ParserWarning)Path)readers) ParserBase)_can_parallelize_csv_default_n_workers_find_chunk_byte_offsets_find_data_start_offset_read_csv_parallelctjjd}tj|j dd||j||j gd|d}|j|dy ) zDWrite a CSV with enough rows to split into multiple parallel chunks.*r'size)foobarbazqux)int_col float_colstr_colFindexN)nprandom default_rngpd DataFrameintegerschoiceto_csv)pathn_rowsrngdfs t/var/www/html/python-recuperauto-other/env/lib/python3.12/site-packages/pandas/tests/io/parser/test_parallel_read.py_make_large_csvr-7sj ))   #C ||AvF|;F+zz">VzL  BIId%I ceZdZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZej$j'dgddZej$j'dgddZdZdZdZdZdZdZej$j'dddej:gdZdZdZ d Z!d!Z"d"Z#d#Z$d$Z%d%Z&d&Z'd'Z(y())TestCanParallelizeCsvz)Unit tests for the eligibility predicate.c Ddddddddddddd }|j||S)NcFrinfer) engineengine_specifiediterator chunksizenrowsskiprowsheader skipfooter index_colusecols compression)update)self overridesbases r,_kwdszTestCanParallelizeCsv._kwdsLs= %"   I r.c|dz }t||jtddt||j sJy)Nbig.csv_PARALLEL_READ_MIN_BYTES)r-setattr_readersr rCr@tmp_path monkeypatchr(s r,test_eligible_large_filez.TestCanParallelizeCsv.test_eligible_large_file]s=)#H&@!D#D$**,777r.c|dz }|jdd|jtddt||j dsJy) zCengine=None (the default) should be treated the same as engine='c'.data.csva,b 1,2 utf-8encodingrFrGNr4 write_textrHrIr rCrJs r,test_accepts_default_enginez1TestCanParallelizeCsv.test_accepts_default_enginedsJ*$  w7H&@!D#D$**D**ABBBr.c|dz }|jddt|d5}t||jrJ dddy#1swYyxYw)NrOrPrQrR)rVopenr rC)r@rKr(fs r,test_rejects_file_like_objectz3TestCanParallelizeCsv.test_rejects_file_like_objectksV*$  w7 $ ) =Q+Atzz|< <<< = = =s A  Acttd|jrJtd|jrJy)Nzhttp://example.com/data.csvzs3://bucket/key.csv)r rC)r@s r,test_rejects_urlz&TestCanParallelizeCsv.test_rejects_urlqs3'(Etzz|TTT'(=tzz|LLLLr.c|dz }|jdt||jdrJt||jdrJy)Nz data.csv.gzfakegzipr>r3 write_bytesr rCr@rKr(s r,test_rejects_compressedz-TestCanParallelizeCsv.test_rejects_compressedusS-' !'djjVj.LMMM'djjWj.MNNNNr.ch|dz }|jdt||jrJy)Nzdata.tarr_rbrds r,test_rejects_tarz&TestCanParallelizeCsv.test_rejects_tar{s5*$ !'djjl;;;;r.ct|dz }|jd|jddi}t||rJy)NrOr_methodr`ra)rcrCr r@rKr(kwdss r,test_rejects_dict_compressionz3TestCanParallelizeCsv.test_rejects_dict_compressionsB*$ !zzx&8z9'd3333r.cp|dz }|jddt||jdrJy)NrOrPrQrRT)r6rVr rCrds r,test_rejects_iterator_modez0TestCanParallelizeCsv.test_rejects_iterator_modes:*$  w7'djj$j.GHHHHr.cp|dz }|jddt||jdrJy)NrOrPrQrR)r7rnrds r,test_rejects_chunksizez,TestCanParallelizeCsv.test_rejects_chunksizes:*$  w7'djj4j.HIIIIr.cp|dz }|jddt||jdrJy)NrOrPrQrRd)r8rnrds r,test_rejects_nrowsz(TestCanParallelizeCsv.test_rejects_nrowss:*$  w7'djjsj.CDDDDr.c|dz }|jddt||jdrJt||jdrJy)NrOrPrQrRpythonrTpyarrowrnrds r,test_rejects_non_c_enginez/TestCanParallelizeCsv.test_rejects_non_c_enginesU*$  w7'djjj.IJJJ'djj j.JKKKKr.cR|dz }|jddt||jdrJt||jdrJ|jtdd t||jd sJt||jdsJy) NrOrPrQrR )lineterminator|rFrG )rVr rCrHrIrJs r,"test_rejects_custom_lineterminatorz8TestCanParallelizeCsv.test_rejects_custom_lineterminators*$  w7(djjj.MNNN'djjj.LMMMH&@!D#D$**D**IJJJ#D$**D**IJJJr.rS)zutf-16zUTF-16 utf_16_lezutf-32 UTF_32_BEc|dz }|jdd|jtddt||j |rJy)NrOrPrQrRrFrGrUr@rKrLrSr(s r,test_rejects_utf16_32_encodingz4TestCanParallelizeCsv.test_rejects_utf16_32_encodingsO*$  w7H&@!D'djj(j.KLLLLr.)latin-1cp1252 shift_jisasciicR|dz }|jdd|jtddt||j |rJt||j dsJt||j dsJt||j dsJy) NrOrPrQrRrFrGUTF8z utf-8-sigrUrs r,test_rejects_non_utf8_encodingz4TestCanParallelizeCsv.test_rejects_non_utf8_encodings *$  w7H&@!D'djj(j.KLLL#D$**g**FGGG#D$**f**EFFF#D$**k**JKKKr.c|dz }|jdd|jtddt||j drJt||j drJt||j d sJy) NrOrPrQrRrFrG;; delimiterz\s+rUrJs r,test_rejects_python_engine_sepsz5TestCanParallelizeCsv.test_rejects_python_engine_sepss*$  w7H&@!D'djj4j.HIII'djj4j.HIII#D$**v**FGGGr.c|dz }|jdd|jtddt||j drJt||j d rJt||j d sJy) NrOrPrQrRrFrG§r») quotechar'rUrJs r,(test_rejects_multibyte_sep_and_quotecharz>TestCanParallelizeCsv.test_rejects_multibyte_sep_and_quotechars*$  w7H&@!D'djj4j.HIII'djj4j.HIII#D$**s**CDDDr.c|dz }|jdd|jtddt||j drJy) NrOrPrQrRrFrG#commentrUrJs r,test_rejects_commentz*TestCanParallelizeCsv.test_rejects_commentsM*$  w7H&@!D'djjj.EFFFFr.c|dz }|jdd|jtddt||j drJy) NrOrPrQrRrFrG\) escapecharrUrJs r,test_rejects_escapecharz-TestCanParallelizeCsv.test_rejects_escapecharsM*$  w7H&@!D'djjDj.IJJJJr.c|dz }|jdd|jtddt||j drJy) NrOrPrQrRrFrGexceldialectrUrJs r,test_rejects_dialectz*TestCanParallelizeCsv.test_rejects_dialectsO*$  w7H&@!D'djjj.IJJJJr.cT|dz }|jdd|jtddt||j dgrJt||j d rJt||j dsJt||j d sJy) NrOrPrQrRrFrGa) parse_datesTFrUrJs r,test_rejects_parse_datesz.TestCanParallelizeCsv.test_rejects_parse_datess*$  w7H&@!D'djjcUj.KLLL'djjTj.JKKK#D$****FGGG#D$****GHHHr. low_memoryFrc|dz }|jdd|jtddt||j |rJt||j dsJt||j sJy) NrOrPrQrRrFrGrTrU)r@rKrLrr(s r,test_rejects_low_memory_falsez3TestCanParallelizeCsv.test_rejects_low_memory_falses}*$  w7H&@!D'djjJj.OPPP#D$****EFFF#D$**,777r.ct|dz }|jddt||jddirJy)NrOrPrQrRanonTstorage_optionsrnrds r,test_rejects_storage_optionsz2TestCanParallelizeCsv.test_rejects_storage_optionssI*$  w7' $**fd^*<    r.cr|dz }|jdd|jtddtj}t ||j |jrJt ||j |jsJt ||j |jsJy)NrOrPrQrRrFrG on_bad_lines) rVrHrIr BadLineHandleMethodr rC BLHM_WARN BLHM_ERROR BLHM_SKIP)r@rKrLr(ris r,test_rejects_on_bad_lines_warnz4TestCanParallelizeCsv.test_rejects_on_bad_lines_warn s*$  w7H&@!D//'djjfFVFVj.WXXX#D$**&BSBS**TUUU#D$**&BRBR**STTTr.c|dz }|jdd|jtddt||j rJy)NrOz a,b 1,2 rQrRrFrGrUrJs r,#test_rejects_blank_line_in_preamblez9TestCanParallelizeCsv.test_rejects_blank_line_in_preamblesJ*$ 9H&@!D'djjl;;;;r.cr|dz }|jddt||jdrJy)NrOrPrQrRc |dk(S)Nr)xs r,zFTestCanParallelizeCsv.test_rejects_callable_skiprows.."s AQRFr.r9rnrds r,test_rejects_callable_skiprowsz4TestCanParallelizeCsv.test_rejects_callable_skiprowss;*$  w7'djjBRj.STTTTr.ct|dz }|jddt||jddgrJy)NrOrPrQrRrrrnrds r,test_rejects_list_skiprowsz0TestCanParallelizeCsv.test_rejects_list_skiprows$s>*$  w7'djj1a&j.IJJJJr.ct|dz }|jddt||jddgrJy)NrOrPrQrRrrGr:rnrds r,test_rejects_multi_level_headerz5TestCanParallelizeCsv.test_rejects_multi_level_header)s>*$  w7'djjAj.GHHHHr.c|dz }|jddt||jdrJt||jdrJy)NrOrPrQrRrr<rrnrds r,test_rejects_index_colz,TestCanParallelizeCsv.test_rejects_index_col.sU*$  w7'djj1j.EFFF'djj3j.GHHHHr.cp|dz }|jddt||jdrJy)NrOrPrQrRFrrnrds r,test_accepts_index_col_falsez2TestCanParallelizeCsv.test_accepts_index_col_false4s:*$  w7'djj5j.IJJJJr.ct|dz }|jddt||jddgrJy)NrOz a,b,c 1,2,3 rQrRrb)r=rnrds r,test_rejects_usecolsz*TestCanParallelizeCsv.test_rejects_usecols;s?*$ (7;'djj#sj.LMMMMr.cl|dz }|jddt||jrJy)Nz small.csvrPrQrRrnrds r,test_rejects_small_filez-TestCanParallelizeCsv.test_rejects_small_file@s5+%  w7'djjl;;;;r.N))__name__ __module__ __qualname____doc__rCrMrWr[r]rergrlrorrruryrpytestmark parametrizerrrrrrrrr False_rrrrrrrrrrrrr.r,r0r0Is03"8C= MO <4 I J E L K [[LMM [[Z)TU LV LHEG K KI [[\E1bii+@A 8B 8  U<U K I I KN z6TestFindChunkByteOffsets._write_csv..us!F1QCq2,!FrQrR)joinrangerV)r@r(r)liness r, _write_csvz#TestFindChunkByteOffsets._write_csvts/"''!Ff !FFF 0r.c|dz }|j|td}tt|d|}|d|k(sJ|dtj j t|k(sJt|dk(sJy)NrOa,b rGrr)rrrrosr(getsize)r@rKr( data_startoffsetss r,test_single_chunkz*TestFindChunkByteOffsets.test_single_chunkxsz*$ ] *3t9aDqzZ'''r{bggooc$i88887|q   r.cx|dz }|j|dtdtjj t |t t |ddk(sJdk(sJtfdDsJtfd ttd z DsJy) NrOrtr)rrrc3>K|]}|cxkxrkncywNr)roffr file_sizes r,rzPTestFindChunkByteOffsets.test_multiple_chunks_cover_full_file..sEc:1 11Esc3:K|]}||dzkywrGNrrrrs r,rzPTestFindChunkByteOffsets.test_multiple_chunks_cover_full_file.."P171:A.PrG) rrrr(rrrallr)r@rKr(rr rs @@@r,$test_multiple_chunks_cover_full_filez=TestFindChunkByteOffsets.test_multiple_chunks_cover_full_files*$ S)] GGOOCI. *3t9aDqzZ'''r{i'''EWEEEEPc'lQ>N8OPPPPr.c|dz }|j|dtd}|j}tt |d|}|ddD]}||dz |dk(rJy) NrOrqrrr rGr )rr read_bytesrr)r@rKr(r raw_bytesrr s r,test_chunks_align_to_newlinesz6TestFindChunkByteOffsets.test_chunks_align_to_newlinessq*$ T*] OO% *3t9aD1R= 5CS1Ws+u4 44 5r.c~|dz }|jdtt|dd}t|dk\sJy)NrOsa,b 1,2 r rr)rcrrr)r@rKr(rs r,/test_fewer_chunks_than_requested_when_file_tinyzHTestFindChunkByteOffsets.test_fewer_chunks_than_requested_when_file_tinys>*$ '*3t9aAF7|q   r.cD|dz }dgdgdzz}|jddddzzd z|jd j|tt |d d t dz dk\sJt fdtt dz DsJy)NrOrs1,2 N s1,"x@s" r. r rrGc3:K|]}||dzkywrrrs r,rzZTestFindChunkByteOffsets.test_one_long_line_does_not_collapse_the_split..rr)insertrcrrrrrr)r@rKr(rrs @r,.test_one_long_line_does_not_collapse_the_splitzGTestFindChunkByteOffsets.test_one_long_line_does_not_collapse_the_splits*$ hZ&00 R$-/&89 %)*3t9bQG7|a2%%%Pc'lQ>N8OPPPPr.N)r r(r r)intreturnNone) rrrrrrrrr'rr.r,rrss"1! Q 5! Qr.rcreZdZdZdZddZdZdZdZdZ dZ d Z d Z d Z d Zd ZdZdZdZdZy)TestReadCsvParallelzAEnd-to-end correctness: parallel result must match serial result.c 2tj|fddi|S)Nr4r2)r#read_csv)r@r(kwargss r, _serial_readz TestReadCsvParallel._serial_reads{{466v66r.c`tt|||}|tjd|S)zBCall the internal helper directly so file-size guards don't apply.z!parallel read fell back to serial)rrrfail)r@r(rk n_workersresults r,_parallel_readz"TestReadCsvParallel._parallel_reads+#CItY? > KK; < r.c ddlm}ddlm}iddd|jddd dd dd dd dd dddddddddddddddddddiddddddddddddddd d!d"dd#d$d%dd&dd'dd(dd)dd*d+d,dd-dddd|jd.}|j ||d|jd|j|j d/|j dd0d1i|j d2|d"3 }|j ||S)4zBReturn kwds as _read() would see them for a default read_csv call.r)lib)_refine_defaults_readr:r3namesr<Nr=dtype converters true_values false_valuesskipinitialspaceFr9r;r8 na_valueskeep_default_naT na_filterskip_blank_linesr date_formatdayfirst cache_datesr6r7r> thousandsdecimal.r|r"quoting doublequoterrrSencoding_errorsstrictrerror)rr memory_mapfloat_precisionr dtype_backendrrrrR) rrr4seprr:defaultsrRr|) pandas._libsr8pandas.io.parsers.readersr9 no_defaultr?pop)r@r(rAr8r9rk kwds_defaultss r, _base_kwdszTestReadCsvParallel._base_kwdssA$C) g) S^^)  )  t ) T ) $ )  4)  D)  )  )  !)  T)  )  t)  ) !) " 4#) $ 4%) & ') ( 4)) * +) , -) . 7/) 0 1) 2 s3) 4 d5) 6 7) 8 q9) : 4;) < $=) > t?) @ A) B xC) D tE) F$## ^^Q) T I-nn.1((7#!3'((?3 01   M" r.c|dz }d}tjjd}tj|j dd||j|d}|j |d|j|}|j||}|j|}tj|jd |y) NrOrrtrrFrTdrop r r!r"r#r$r%r'rZr6r1tmassert_frame_equal reset_index r@rKr(nr*r+rkr5expecteds r,test_ints_and_floatsz(TestReadCsvParallel.test_ints_and_floatss*$ ii##A& \\ QQ 7cjjmL M $e $t$$$T40$$T* f00d0;XFr.c|dz }d}tjjd}tj|j gd||j dd|d}|j|d |j|}|j||}|j|}tj|jd |y) NrOr\rG)alicebobcarolrr )namevalFrTr^)r r!r"r#r$r&r%r'rZr6r1rarbrcrds r, test_stringsz TestReadCsvParallel.test_stringss*$ ii##A& \\ #.Js)!PaCs!A3aAwb"9!Psr\rQrRr rrrVrZrrr@rKr(rrks r, test_implicit_index_returns_nonez4TestReadCsvParallel.test_implicit_index_returns_noneEsb.("''!P5.spy[s%   X &+Hh RTR Rr.2pandas.io.parsers.readers._find_chunk_byte_offsetsr Tr^rr ) rrVrrHrZrrr1rarbrc) r@rKrLr(rrrrkr5rfrs @r,%test_long_first_line_keeps_fine_splitz9TestReadCsvParallel.test_long_first_line_keeps_fine_splitOs *$Cf ~R01 v7AQCr!B-77 9  S PRUVt$#CItQ7!!!$$T* f00d0;XF|r!!!#8sCc |dz }djdtdDdz}djdgdzdz}|j||dzzd |j|}t t ||d Jy) Nwide.csvrc3&K|] }d| ywr2Nrrs r,rzKTestReadCsvParallel.test_few_estimated_rows_returns_none..os6aAaS'6r~ 123456789,rQrRr r~)r@rKr(r:rowrks r,$test_few_estimated_rows_returns_nonez8TestReadCsvParallel.test_few_estimated_rows_returns_noneis *$65:66=hh }s*+d2 s*W=t$!#d)T15===r.c |dz }|jddjdtdDzd|j|d}t t ||d Jy) Nblank_first.csvr~rc32K|]}|d|dzdywrrr~Nrrs r,rzNTestReadCsvParallel.test_blank_first_data_line_returns_none..{s!BaS!a%+Br\rQrRrr )rVrrrZrrrjs r,'test_blank_first_data_line_returns_nonez;TestReadCsvParallel.test_blank_first_data_line_returns_noneusj++  277BU5\BB BW  tD1!#d)T15===r.c|dz }ddjdtdDz}|j|d|j|}t t ||dJy) Nhdr.csvz "col 1",col2 rc3.K|] }d|d|dywrz,yr~Nrrs r,rzQTestReadCsvParallel.test_quoted_newline_in_header_returns_none..s+Qqas"QCrN+Qsr\rQrRr r~rs r,*test_quoted_newline_in_header_returns_nonez>TestReadCsvParallel.test_quoted_newline_in_header_returns_nonesb)#!BGG+QE%L+Q$QQ 0t$!#d)T15===r.c"|dz }djdtdD}|jd|z|j|}t t ||dJ|jd|zt t ||dJy)Nzcr.csvr.c3JK|]}d|d|djywrencoders r,rzLTestReadCsvParallel.test_bare_cr_before_data_returns_none..s'FA!A3b2--/F!#r\s a,b c,d r s a,b c,d )rrrcrZrrr@rKr(bodyrks r,%test_bare_cr_before_data_returns_nonez9TestReadCsvParallel.test_bare_cr_before_data_returns_nones("xxFuFF 4/0t$!#d)T15=== .56!#d)T15AAAr.c|dz }d}tjjd}tj|j dd||j dd|d}|j |d|j|d d d }|j||}|j|d d d }tj|jd |y) NrOr\rrrtr]Frfloat64r;Tr^r`rds r,test_dtype_specifiedz(TestReadCsvParallel.test_dtype_specifieds*$ ii##A& \\ QQ 7cll1cST>UV W $e $t+KL$$T40$$Tyy1Q$R f00d0;XFr.cl|dz }djdtdD}|jd|zd|j|dti }t t ||d J|jd |zd|j|dti }t t ||d Jy) NrOrc32K|]}|d|dzdywrrGr~Nrrs r,rzQTestReadCsvParallel.test_dict_dtype_duplicate_names_falls_back..s!>A!Aa!eWB'>rr\za,a rQrRrrr r)rrrVrZrrrs r,*test_dict_dtype_duplicate_names_falls_backz>TestReadCsvParallel.test_dict_dtype_duplicate_names_falls_backs*$ww>u>> $9tC:6!#d)T15=== $9tC:6!#d)T15AAAr.N)r )rrrrr1r6rZrgrnrtrrxrrrrrrrrrr.r,r-r-s^K7>H GG G GG>"4 > >> B G Br.r-zcsv_bytes,expected_rowss#a,b,c 1,"hello world",3 4,normal,6 rsimple_embedded_newline)ids(a,b,c 1,"hello 2,foo,bar",3 100,baz,200 +adversarial_after_split_looks_like_full_rows5a,b 1,"line1 line2 line3" 2,normal 3,"also embedded" rvmultiple_embedded_newlinescZ|jdd|dz }|j|tjt j |}tj|}t ||k(sJtj|jd|jdy)a% CSV files with newlines inside quoted fields must produce the same result as a direct serial read. The parallel path either parses correctly (when the split misses the embedded newline) or falls back to serial (when it hits it). Silent data corruption must never occur. 2pandas.io.parsers.readers._PARALLEL_READ_MIN_BYTESrGz embedded.csvTr^N) rHrcr#r/ioBytesIOrrarbrc)rKrL csv_bytes expected_rowsr(serialparallels r,*test_embedded_newline_falls_back_to_serialrsBLaP n $DY [[I. /F{{4 H x=M )) )$');););)Fr.c,|dz }t||jtddtjdd5tj |}dddtj |d}t j|y#1swY7xYw) z read_csv() transparently uses the parallel path for large local files. Result must match the serial read obtained via engine='python'. rErFrGmode.max_threadsr NrwrT)r-rHrIr#option_contextr/rarbrKrLr(r5rfs r,test_read_csv_auto_parallelrs i DD"The parallel path is on by default on every threaded platform.rErFrG cpu_countcyNr rrr.r,rz-test_parallel_on_by_default..%r.available_cpu_countcyr rrr.r,rz-test_parallel_on_by_default..&rr.cNj|tjSr rr#r$rr0callss r,stubz)test_parallel_on_by_default..stub- T||~r.rplatformN)r-rHrIrsysr#r/r)rKrLrr(rrs @r,test_parallel_on_by_defaultrs i DD".=rr.rcyr rrr.r,rz3test_parallel_default_off_on_wasm..>rr.cNj|tjSr rrs r,rz/test_parallel_default_off_on_wasm..stubBrr.rr emscriptenN)r-rHrIrrr#r/)rKrLr(rrs @r,!test_parallel_default_off_on_wasmr8s i DD".Trr.rcyr rrr.r,rz2test_parallel_default_thread_cap..Wrr.cNj|tjSr r)_pathrCr4workerss r,rz.test_parallel_default_thread_cap..stub[sy!||~r.rrrN) r-rHrIrrr#r/clearr)rKrLr(rrs @r, test_parallel_default_thread_caprMs i DD"> MMO  -q 1 D qc>>s %D  DceZdZdZej j dgdej j dgddZdZdZ ej j dd d gd Z y ) TestDefaultNWorkersz(Unit tests for the default worker count.rrzcpu_count, available, expected))rNr)rNr)rrGrG)rrr)rrr)rrrcftjdk(sJ|jtjd||jtjdfd|jtdfdt j dd5t|k(sJ dddy#1swYyxYw)NrrrcSr r)rsr,rzFTestDefaultNWorkers.test_combines_cap_and_allocation..sir.rcSr r) availablesr,rzFTestDefaultNWorkers.test_combines_cap_and_allocation..sYr.r)rI_MAX_DEFAULT_WORKERSrHrrr#rr)r@rLrrrfrs `` r, test_combines_cap_and_allocationz4TestDefaultNWorkers.test_combines_cap_and_allocationss",,111HLL*mDHKK6GHH&;=NO   14 8 4%'83 33 4 4 4s B''B0c|jtjddtjdd5t dk(sJ dddy#1swYyxYw)NrrrrrGrHrIrr#rrr@rLs r,test_wasm_is_serialz'TestDefaultNWorkers.test_wasm_is_serialsOHLL*lC   11 5 -%'1, ,, - - - AAc0|jtjdd|jtjdd|jtddt j dd5t dk(sJ dddy#1swYyxYw) Nrrrcy)Nrrrr.r,rzBTestDefaultNWorkers.test_max_threads_exceeds_cap..rr.rcyNrGrrr.r,rzBTestDefaultNWorkers.test_max_threads_exceeds_cap..rr.rr#)rHrIrrr#rrrs r,test_max_threads_exceeds_capz0TestDefaultNWorkers.test_max_threads_exceeds_capsz HLL*g>HKKi@H&;YG   12 6 .%'2- -- . . .s 2B  Brrc|jtjd|tjdd5t dk(sJ dddy#1swYyxYw)Nrrrvr)r@rLrs r,test_max_threads_winsz)TestDefaultNWorkers.test_max_threads_winssOHLL*mD   11 5 -%'1, ,, - - -rN) rrrrrrrrrrr rr.r,rrps~2 [[_.JK [[(   4 L 4- . [[_w.@A-B-r.rc |jddtjdd5tj|fi|cdddS#1swYyxYw)z@read_csv with the parallel path force-enabled on every platform.rrGrr N)rHr#rr/)r(rLr0s r,_read_forced_parallelr sJLaP  -q 1+{{4*6*+++s A  Ac<ddjdtdDz}|dz }|j|t||d}t j t j|d}tj|||jdd k(sJy) N col1,col2 r.c3LK|]}d|d|djdyw)useñorrr~rNrrs r,rz6test_parallel_latin1_matches_serial..s/$34(1#Qqc##I.$s"$rz latin.csvrrRrcol1useñor0 rrrcr r#r/rrrarblocrKrLrawr(r5rfs r,#test_parallel_latin1_matches_serialrs 388$8=c $ C k !DS "4y IF{{2::c?Y?H&(+ ::i K // /r.cddjdtdDzdzdjdtddDz}|d z }|j|d }tjt | 5t ||d dddtjt | 5tjtj|d dddy#1swYXxYw#1swYyxYw)Nrr.c3HK|]}|d|djywz,xr~Nrrs r,rzDtest_parallel_ascii_non_ascii_byte_matches_serial..s%>aaS1#R='')> "rs500,é c3HK|]}|d|djywrrrs r,rzDtest_parallel_ascii_non_ascii_byte_matches_serial..s%DaaS1#R='')Drirqz ascii.csvz'ascii' codec can't decoderrrR) rrrcrrUnicodeDecodeErrorr r#r/rrrKrLrr(rs r,1test_parallel_ascii_non_ascii_byte_matches_serialrs  ((>5:> > ?   ((D5d3CD D E k !DS &C ) 5CdK'BC ) 57 BJJsOg677CC77s>C$0+C0$C-0C9cnddjdtdDz}|dz }|j|t||dti}t j tj|dti}tj|||jd|jdk(sJy) Nsa,a r.c3NK|]}|d|dzdjywrrrs r,rzDtest_parallel_dup_names_dict_dtype_matches_serial..*LA1QUG2.557L#%rzdup.csvrrza.1) rrrcr rr#r/rrrarbdtypesrs r,1test_parallel_dup_names_dict_dtype_matches_serialr$s SXXLsLL LC i DS "4S#J GF{{2::c?3*=H&(+ == u!5 55 5r.cXddjdtdDz}|dz }|j|t||d}t j t j|d}tj|||djtjk(sJy) Ns# a comment col1,col2 r.c3NK|]}|d|dzdjywrrrs r,rzEtest_parallel_comment_before_header_matches_serial..s/1&'1#Qq1ugR!1r"rz comment.csvrrr rrrcr r#r/rrrarbr;r int64rs r,2test_parallel_comment_before_header_matches_serialr)s &1+0:1) C m #DS "4c BF{{2::c?C8H&(+ &>  288 ++ +r.cPddjdtdDz}|dz }|j|t||}t j t j|}tj|||djtjk(sJy)Ns col1,col2 r.c3NK|]}|d|dzdjywrrrs r,rzHtest_parallel_blank_line_before_header_matches_serial..s*%TA1QUG2&6&=&=&?%Tr"rz blank.csvrr'rs r,5test_parallel_blank_line_before_header_matches_serialr,s SXX%Ts%TT TC k !DS "4 5F{{2::c?+H&(+ &>  288 ++ +r.z head,kwargss a,"line break",c FIRST,ROW,HERE sa,b,c FIRST,ROW,HERE sjunk a,b,c FIRST,ROW,HERE r9rGsjunk FIRST,ROW,HERE )r9r:cX|djdtdDz}|dz }|j|t||fi|}t j t j|fi|}tj|||jdjgdk(sJy)Nr.c3PK|]}d|d|d|dj yw)sz,tz,ur~Nrrs r,rzCtest_parallel_bare_cr_before_data_matches_serial.. s.K1AaS1#Rs"-446K$&rz bare_cr.csvr)FIRSTROWHERE) rrrcr r#r/rrrarbiloctolist)rKrLheadr0rr(r5rfs r,0test_parallel_bare_cr_before_data_matches_serialr7s Kc KK KC m #DS "4 ? ?F{{2::c?5f5H&(+ ;;q> "&> >> >r.zpreamble,skiprows))rr)s r)s junk1 junk2 rc:|djdtdDz}|dz }|j|t||d|}t j t j|d|}tj|||jdk(sJy)Nr.c3NK|]}|d|dzdjywrrrs r,rzEtest_parallel_blank_first_data_line_matches_serial..&r!r"rrr)rr) rrrcr r#r/rrrarbshape)rKrLpreambler9rr(r5rfs r,2test_parallel_blank_first_data_line_matches_serialr<s SXXLsLL LC ' 'DS "4TH UF{{2::c?4(KH&(+ <<8 ## #r.c|dz }|jdgfd}|jd|d}tjt|5t ||dddddgk(sJdy#1swYxYw) Nzblank_only.csvs cDt|||}j||Sr )rr)rrkr4r5returneds r,rz1test_parallel_all_blank_lines_raises..spy:s"#HdI> r.z,pandas.io.parsers.readers._read_csv_parallelzNo columns to parse from filerrz#the parallel path did not fall back)rcrHrrrr )rKrLr(rrr?s @r,$test_parallel_all_blank_lines_raisesr@0s & &D[!H FL )C ~S 1>dK=> v DDD  >>s A33A<cdddzzdz}|dz }|j|t||}tjt j |}t j||y)Nrar\s,bz longline.csv)rcr r#r/rrrarbrs r,test_parallel_single_long_linerCIsa 4%< '% /C n $DS "4 5F{{2::c?+H&(+r.z.WASM cannot spawn threads, so no split happensc|dz }dddzzdz}|jdddzz|zdd zzd gfd }|jd d|jd|tjdd5tj|}dddtjdd5tj|}dddt j sJdddk\sJy#1swYgxYw#1swY?xYw)NrOz1,9i;r~rz1,2 r\rrQrRc^t|||g|}jt|dz |Sr)rrr)rrrrr chunk_countss r,rz4test_parallel_long_line_keeps_the_split..spycs2*8XzQDQCL1,-r.rrGrrr the parallel path did not runrrv)rVrHr#rr/rarb)rKrLr(blobrrfr5rGs @r,'test_parallel_long_line_keeps_the_splitrJUs j D #,  %DOO'E/!D(7V+;;gL LaPLcR  -q 1%;;t$%  -q 1#T"#&(+ 888< ?a  %%##s5C1)C=1C:=DcDdjdtdD}d|zdz|z}|dz }|j|t||}t j t j|}tj|||jddk(sJy) Nr.c3HK|]}|d|djywrrrs r,rzBtest_parallel_mixed_dtype_column_matches_serial..{s%?qs!A3b\((*?rrrs oops,999 z mixed.csvr0r)rKrLcleanrr(r5rfs r,/test_parallel_mixed_dtype_column_matches_serialrOvs HH?E#J? ?E 5 = 05 8C k !DS "4 5F{{2::c?+H&(+ ::i C '' 'r.ctjd|dz }djdtdD}|j d|zdzd|j d d t jd d 5t j|d }dddt jd d 5t j|d }dddtjy#1swYVxYw#1swY,xYw)NrxrOr~c32K|]}d|dzd|yw)r/ rNrrs r,rzAtest_parallel_deferred_strings_pyarrow_backend..s!;!qR!%;rr\rrQrRrrGrrRr ) r importorskiprrrVrHr#rr/rarbrKrLr(rowsrrs r,.test_parallel_deferred_strings_pyarrow_backendrWs   " j D 99;uT{; ;DOOGdNT)GO<LaP  -q 1<T;<  -q 1>;;t9=>(F+ <<>>s=C*3C6*C36C?zDignore:The 'future.infer_string' option:pandas.errors.Pandas4Warningc  tjdgd}|Dcgc]#}djdt|D%}}|dz }djd|dzD}|j d|zd |j d d t jd ddddd 5t j|}dddt jd ddddd5t j|}dddtj|djt jdtjk(sJ|dj|dzk(sJycc}w#1swYxYw#1swYxYw)Nrx) rGrrr#!@rc3PK|]}ttd|dzz yw)rN)chrord)rposs r,rzCtest_parallel_deferred_strings_token_width_tiers..s"ASCHsRx'(Ar0z tiers.csvc3HK|]}|d|jdywr)upper)rvalues r,rzCtest_parallel_deferred_strings_token_width_tiers..s#JEeWAekkm_B/JrirrQrRrrGzfuture.infer_stringTmode.string_storagerr r)na_value)rrTrrrVrHr#rr/rarbr; StringDtyper rpr5) rKrLwidthswidthvaluesr(rVrrs r,0test_parallel_deferred_strings_token_width_tiersrlsy  " /FOUFKAE%LAAF k !D 77JVc\J JDOOGdNWO5LaP     #T"#     %;;t$%(F+ #;  y266 J JJ J #;   6C< // /C##%%s(E)*E."E:.E7:FcdjdtdD}d|zdz|z}|dz }|j|d|jd d t j d d 5t j |}dddt j d d 5t j |}dddtjy#1swYTxYw#1swY,xYw) Nrc3,K|] }|d|dywrrrs r,rzDtest_parallel_deferred_strings_mixed_chunk_dtypes..s6Qqc1#RL6riz col1,col2 ztext,7 z mixed_str.csvrQrRrrGrr  rrrVrHr#rr/rarb)rKrLchunkrr(rrs r,1test_parallel_deferred_strings_mixed_chunk_dtypesrqs GG6%+6 6E % * ,u 4C o %DOOC'O*LaP  -q 1#T"#  -q 1%;;t$%(F+ ##%%s-C!C"C"C+rFc^dddzz}|dz }|j|t|||}tjt j ||}t j|||ddk(jdk(sJ|djjdk(sJy) Ns col hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello snan 9223372036854775808 z uint64.csvrcolrpr) rcr r#r/rrrarbsumisna)rKrLrrr(r5rfs r,-test_parallel_low_memory_false_matches_serialrws &)LPT(T TC l "DS "4 LF{{2::c?zBH&(+ 5MU " ' ' )Q .. . %=    # # % -- -r.r0r@MISSINGrtrRnumpy_nullablec<|jdrdnd}|ddz}|dz }|jd|zd t||fi|}tjd d 5tj |fi|}dddt j|y#1swY xYw) Nr@rxrpz 9223372036854775808 rszuint64_default_lm.csv col text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text text rQrRrrG)getrVr r#rr/rarb)rKrLr0tokenrr(r5rfs r,/test_parallel_uint64_na_conflict_matches_serialr~s  K0IeEW+ ,t 3D - -DOO,t3gOF "4 ? ?F  -q 1/;;t.v./&(+//s BBcddz}|dz }|jd|zdgtjfd}|jtd|t ||d }d gk(sJt j d d 5t j|d }dddtj|y#1swY xYw) Nznan 9223372036854775808 rszuint64_no_filter.csvr{rQrRc>|i|}j|du|Sr r)rr0r5 fell_backorigs r,rz=test_parallel_uint64_na_conflict_na_filter_false..spys)t&v&4( r.rF)rBrrG) rVrIrrHr r#rr/rarb) rKrLrr(rr5rfrrs @@r,0test_parallel_uint64_na_conflict_na_filter_falser s ($ .D , ,DOO,t3gOFI  & &D "6< "4 FF     -q 16;;tu56&(+66s >B55B>c,ddjdtdDz}|dz }|j|t||}t j t j|}tj||t|dk(sJy)Ns "col 1",col2 r.c3JK|]}d|d|djywrrrs r,rzHtest_parallel_quoted_newline_in_header_matches_serial..'s''TA!A3b2(=(=(?'Trrr) rrrcr r#r/rrrarbrrs r,5test_parallel_quoted_newline_in_header_matches_serialr"s} sxx'Ts'TT TC i DS "4 5F{{2::c?+H&(+ v;#  r.cddjdtdDz}|dz }|j|d}tjt |5t ||dd i dddy#1swYyxYw) Nrr.c3HK|]}|d|djywrrrs r,rz7test_parallel_storage_options_raises..4s%H1QCrl113Hrrzso.csvz?storage_options passed with file object or non-fsspec file pathrrTr)rrrcrrrr rs r,$test_parallel_storage_options_raisesr1su SXXHU3ZHH HC h DS KC z -QdK&$PQQQs A22A;c ddjdtdDzdzdjdtddDz}|dz }|j|tjt d 5t ||d }dddtjt d 5tjtj|d }dddtjy#1swYnxYw#1swY,xYw) Nrr.c3HK|]}|d|djywrrrs r,rz?test_parallel_on_bad_lines_warn_line_numbers..Bs%=QaS!B<&&(=rrs1,2,3,4 c3HK|]}|d|djywrrrs r,rz?test_parallel_on_bad_lines_warn_line_numbers..Ds%BQaS!B<&&(BrXbad.csvzline 302rwarnr rrrcraassert_produces_warningr r r#r/rrrbrs r,,test_parallel_on_bad_lines_warn_line_numbersr=s  ((=%*= = >   ((B%S/B B C i DS # #M DO&t[vNO # #M DE;;rzz#VDE&(+ OOEEs<C8.+D8DD cJddjdtdDz}|dz }|j|t||}t j t j|}tj||t|jddddgk(sJy) Nrr.c3VK|]!}d|d|d|dzdj#ywr|rrs r,rz>test_parallel_implicit_index_matches_serial..Ts7$-.#aS!Aa!eWB&&($s')rrzridx0idx1) rrrcr r#r/rrrarblistrrs r,+test_parallel_implicit_index_matches_serialrPs 388$27*$ C n $DS "4 5F{{2::c?+H&(+  Ra !ff%5 55 5r.cnGddtj}ddjdtdDz}|dz }|j |t |||}t jtj||}tj|||jd d k(sJy) Nc<eZdZdZdZdZdZdZejZ dZ y)Ctest_parallel_dialect_escapechar_matches_serial..EscDialectrrJTFz rN) rrrrrrLr?r|csv QUOTE_NONErKrrr.r, EscDialectres*    .. r.rrr.c3JK|]}d|d|djyw)z text\ morerr~Nrrs r,rzBtest_parallel_dialect_escapechar_matches_serial..ns-$./,qc1#R '')$rrz dialect.csvrrz text more0) rDialectrrrcr r#r/rrrarbr)rKrLrrr(r5rfs r,/test_parallel_dialect_escapechar_matches_serialr`s S[[ 388$38:$ C m #DS "4j IF{{2::c?J?H&(+ ::i M 11 1r.cddjdtdDz}|dz }|j|d}tjt |5t ||d }dddtjt |5tjtj|d }dddtjy#1swYnxYw#1swY,xYw) Ns col1;;col2 r.c3NK|]}|d|dzdjyw)rrr~Nrrs r,rz=test_parallel_multichar_sep_matches_serial..~s*$TAs"QUG2%6%=%=%?$Tr"rz multisep.csv#Falling back to the 'python' enginerr)rSr)rKrLrr(warn_msgr5rfs r,*test_parallel_multichar_sep_matches_serialrzs CHH$Ts$TT TC n $DS4H # #M BD&t[dCD # #M B:;;rzz#D9:&(+ DD::sC +C C C)c:tdDcgc]}|dd|dzd}}|jddj|zdzjd|j dd t t |d d }tt |d |d }|d }|j}|jd|} | |z t|k(sJt|d5} | j|| j|ddd|Scc}w#1swY|SxYw)ayWrite a fixed-width CSV, then overwrite the line at the second chunk boundary with *replacement* (same byte length, so offsets stay valid). Returns the boundary, for :func:`_assert_chunk_starts_at` to check against the split the read actually plans - hard-coding a count here that the read does not use would put the line in the middle of a chunk instead. 06drrrr~rQ2pandas.io.parsers.readers._PARALLEL_MIN_CHUNK_ROWSrGr r r4rr+bN)rrcrrrHrrrrrrrYseekwrite) r( replacementrLrrVrrboundaryrline_endfds r,_write_with_line_at_chunk_startrs-2$K 8qqgQq1uSk " 8D 8 g $/$6>>wGHLaP(TAq9J's4y"jANGqzH // Cyy)H h #k"2 22 2 dE b   O' 9  OsD #DDcJgtfd}|jd|S)z9Record the byte offsets each planned split actually uses.c@|||g|}j||Sr r)rrrrrrealseens r,rz"_spy_on_chunk_offsets..spys'x:== Gr.r)rrH)rLrrrs @@r,_spy_on_chunk_offsetsrs*D #D LcR Kr.cB|sJd||dvsJ|d|dy)NrHrz is not a chunk start in r)rrs r,_assert_chunk_starts_atrs9 0004 tAw O8*,Ed1gY OO r.c|dz }t|d|}t|}tjtd5t ||dddt ||y#1swYxYw)N ragged.csv 11,22,33,4444zExpected 2 fieldsr)rrrrrr r)rKrLr(rrs r,/test_parallel_ragged_line_at_chunk_start_raisesrs_ l "D.t5E{SH  -D {*= >1dK01D(+11s AA$ch|dz }t|d|}t|}t|}tjdd5tj |d}dddt ||d}tj|t|dk(sJt|||dgk(sJy#1swYRxYw) NrrrrGskiprir) rrrr#rr/r rarbrr)rKrLr(rrrrfr5s r,>wGH(TAq9J'D 1jANqQH dE +b 1  )*+ ! -D  -q 1/;;t.v./LaP  -q 1-T,V,-&(+ F3K$6$6 66 6D(+# ;++ //--s)F&F )F0F" FF"F+c dgtdDcgc] }dd|d|zc}z}|dz }|jdj|dzd tjd d 5tj |}dddt |}t||}tj||d jd dk(jsJ|dgk(sJycc}w#1swYixYw)Nza,brurrzbom.csvr~rQrRrrGrrr) rrVrr#rr/rr rarbrr)rKrLrrr(rfrr5s r,-test_parallel_bom_bytes_mid_file_match_serialrs GuT{C!xAaS!+-C CE i DOODIIe$t+gO>  -q 1%;;t$%{+H "4 5F&(+ 3KOOA ( * / / 11 1 x  D%%sC 'C%%C.cdjdtdD}|dz }|jd|jzt j dd5t j |ddd g }dddt||ddd g }tj||djd d k(sJy#1swYHxYw) Nrc32K|]}|d|dzdywrrrs r,rz>test_parallel_bom_at_file_start_header_none..s!9aS!a%#9rrzbom0.csvsrrGrr)r:r:r) rrrcrr#rr/r rarbr4)rKrLrr(rfr5s r,+test_parallel_bom_at_file_start_header_noners 779U4[9 9D j D_t{{}45  -q 1D;;tDc CD "4T#s TF&(+ #;  A ! ## # DDs CC c(|dz }djdtdD}|jd|zdzd|jdd t j d d 5t j d d 5t j |}dddt j d d 5t j |}ddddddtjy#1swY\xYw#1swY4xYw#1swY8xYw)NrOr~c30K|]}|d|dzyw)rrNrrs r,rz.#s:s"QUG_:r\rrQrRrrGrfrwrr rorUs r,)test_parallel_string_dtype_python_storagers j D 99:eDk: :DOOGdNT)GO<LaP  0( ;)   11 5 '[[&F '   11 5 ){{4(H )) (F+  ' ' ) )))s<(D?C0D3C< D0C9 5D<D DDc|dz }djdtdD}|jd|zdzddzz|zdzd|jd d t j d d 5t j |}dddt j d d 5t j |}dddtjy#1swYTxYw#1swY,xYw) NrOr~c30K|]}|d|dzywrrNrrs r,rz/test_parallel_blank_line_run..48s!AE7^8rrtrrrQrRrrGrr ro)rKrLr(rVrfr5s r,test_parallel_blank_line_runr/s j D 998U3Z8 8DOO$u ,t3d:WLaP  -q 1%;;t$%  -q 1#T"#&(+ %%##s4C(C)C&)C2cX|dz }djdtdD}djdtdD}|jd|zdz|zdz|zdzd |jd d |jd d t j d d 5t j |d}dddt j d d5t j |d}dddtjy#1swYVxYw#1swY,xYw)NrOr~c30K|]}|d|dzywrrrs r,rz2test_parallel_bad_line_run_skip..Errrtc3 K|]}dyw)z x,y,z,w,vNr)r_s r,rz2test_parallel_bad_line_run_skip..Fs5AK5s r\rrQrRrrGrrrrr ro)rKrLr(rVbadrfr5s r,test_parallel_bad_line_run_skiprAs  j D 998U3Z8 8D ))5t5 5COOGdNT)C/$6=DwOWLaPLaP  -q 1:;;t&9:  -q 18T78&(+ ::88s'DD D D)cH|dz }|jd|jddtjdd5tj|}dddtj|d}t j ||d d d k(sJy#1swYDxYw) Nz nul_bool.csvsa True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True True "Truexyz" False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False False rrGrr rwrTrrqzTruexyz)rcrHr#rr/rarbrs r,)test_parallel_embedded_nul_boolean_columnrUs n $DUVLaP  -q 1#T"#{{41H&(+ #;s } ,, , ##s BB!ctjd5}tjd||i|}ddd||fS#1swYfSxYw)zBRun *func*, returning every warning it raises (no de-duplication).TrecordalwaysN)warningscatch_warnings simplefilter)funcrr0recordedr5s r,_warnings_fromrdsY   -'h't&v&' 8 ' 8 s AA chgtjfd}|jtd|S)zARecord how each parallel attempt ends: used, declined, or raised.c |i|}j|d|Sd|S#t$rjdwxYw)Nraisedrdeclined) Exceptionr)rr0r5originalrs r,trackedz _track_parallel..trackedqs] t.v.F &"4E ;EE   OOH %  s (Ar)rIrrH)rLrrrs @@r,rrls2H**H"6@ Or.cd|dS)Nz5Both a converter and dtype were specified for column z# - only the converter will be used.r)rls r,_converter_dtype_warningr~s ?vF+ +r.r:rcol2z2WASM stays serial, so no chunk repeats the warningcddjdtdDz}|dz }|j|tj |t tj |dd}t |}tt||fi|\}}ttjtj|fi|\} } |dgk(sJtj|| |D cgc]} t| j c} |D cgc] } t#| c} k(sJ|d j$t&usJ|d j(t*k(sJycc} wcc} w) Nrr.c3NK|]}|d|dzdjywrrrs r,rz;test_parallel_converter_dtype_warns_once..*#R!qc1q5'$4$;$;$=#Rr"rz converter.csvr(r<r;rr)rrrcdictfromkeysr)rrr r#r/rrrarbrmessagercategoryr filename__file__) rKrLr:rr(r0rr5rrfrwarningrls r,(test_parallel_converter_dtype_warns_oncersD 388#RuSz#RR RC o %DSmmE3/ug.F{+H%t[4:FH!bjjoHHKHa x  &(+08 9WC 938>+/ &>   A;  = 00 0 A;  8 ++ + :>s E Ez+WASM stays serial, so the gather never runscTddjdtdDz}|dz }|j|d}|jtd|dt idd id }t |}tjd 5}tjd tjtd5t||fi|dddddd|dgk(sJDcgc]}t|jc}t!dgk(sJy#1swYLxYw#1swYPxYwcc}w)Nrr.c3NK|]}|d|dzdjywrrrs r,rz=test_parallel_gather_exception_still_warns..*#S!qc1q5'$4$;$;$=#Sr"rqboom.csvctd)Nboom RuntimeError)rr0s r,rz8test_parallel_gather_exception_still_warns..booms 6""r._concatenate_chunksrr(rTrrrrr)rrrcrHrIr)rrrrrrrr rrr rKrLrr(rr0rrrs r,*test_parallel_gather_exception_still_warnsrs 388#SuT{#SS SC j DS#"7>#SMVW4E FF{+H   -?h' ]]v > ?? z !! !08 9WC 9 (>    ? ??? :s*1D4D DD% D DD"z&WASM stays serial, so no worker raisesc(ddjdtdDz}|dz }|j|d}t|ddd id }t |}t j d 5}t jd tjtd5t||fi|dddddd|dgk(sJDcgc]}t|jc}tdgk(sJy#1swYLxYw#1swYPxYwcc}w)Nrr.c3NK|]}|d|dzdjywrrrs r,rzFtest_parallel_worker_exception_falls_back_to_serial..rr"rqrc&|dk(r td|SN1400rrres r,rzAtest_parallel_worker_exception_falls_back_to_serial..boom F?v& & r.)rrrr(rTrrrrr)rrrcr)rrrrrrrr rrrrs r,3test_parallel_worker_exception_falls_back_to_serialrs  388#SuT{#SS SC j DS &)$767BS TF{+H   -?h' ]]v > ??  | ## #08 9WC 9 (>    ? ??? :s*-1DC7,DD7D <DD c&tdDcgc] }|d| }}d|d<ddj|zdzj}|dz }|j|t j t d 5}tjtj|d d i dddt j t tj 5t||d d i dddycc}w#1swYVxYw#1swYyxYw) Nrrzx,3000rsrr~rz position 3000rrInt64r)rrrrcrrrr#r/rrrrer )rKrLrrVrr(rfs r,*test_parallel_bad_value_raises_like_serialrs!&d ,1qc1#J ,D ,DJ TYYt_ $t + 3 3 5C i DS z 9;X BJJsOC>:; zX^^)< =GdKW~FGG - ;;GGsC69-C;D;DDcddjdtdDz}|dz }|j|gGfddtj}|j td|d }t j td 5t||d |i dddtdk(sJdjsJy#1swY+xYw)Nrr.c3NK|]}|d|dzdjywrrrs r,rz>test_parallel_chunk_failure_unmaps_the_file..rr"rqrceZdZdfd Zy)Atest_parallel_chunk_failure_unmaps_the_file..TrackingMmapc(j|yr r)r@rr0mappeds r,__init__zJtest_parallel_chunk_failure_unmaps_the_file..TrackingMmap.__init__s MM$ r.Nr*r+)rrrr)rsr, TrackingMmaprs r.rmmapc&|dk(r td|Sr rr s r,rz9test_parallel_chunk_failure_unmaps_the_file..boomrr.rrr)r<rGr) rrrcrrHrrrr rclosed)rKrLrr(rrrs @r,+test_parallel_chunk_failure_unmaps_the_filers 388#SuT{#SS SC j DS F tyy fl3 |6 2LdKVTNKL v;!   !9    LLs C  Cz4WASM stays serial, so there is no attempt to declinec"ddjdtdDzdjdtdDz}|dz }|j|dtidd id }t |}t t ||fi|\}}t tjtj|fi|\}} |d gk(sJtj|||D cgc]} t| jc} tdgk(sJycc} w) Nrr.c3NK|]}|d|dzdjywrrrs r,rzItest_parallel_fallback_does_not_repeat_c_layer_warning.. s*AaS!a%#**,Ar"ic3HK|]}|d|djywrrrs r,rzItest_parallel_fallback_does_not_repeat_c_layer_warning..s%=aaS1#R='')=r2z fallback.csvrr(rr)rrrcr)rrr r#r/rrrarbrrr) rKrLrr(r0rr5rrfrrs r,6test_parallel_fallback_does_not_repeat_c_layer_warningr$s  ((AeCjA A B ((=59= =  > n $DS#SMVW4E FF{+H%t[4:FH!bjjoHHKHa  | ## #&(+08 9WC 9 (>   9sD z6WASM stays serial, so there is no attempt to fall backcddjdtdDz}|dz }|j|t|}t t ||d\}}t t jtj|d\}}|dgk(sJtj|||D cgc]} t| jc} d gk(sJycc} w) Nrr.c3ZK|]#}|d|dzd|dzdj%yw)rrrvr~Nrrs r,rzNtest_parallel_fallback_does_not_repeat_python_layer_warning..)s9$./1#Qq1ugQq1ugR '')$s)+rz mismatch.csvFrrzkLength of header or names does not match length of data. This leads to a loss of data with index_col=False.)rrrcrrr r#r/rrrarbrr) rKrLrr(rr5rrfrrs r,;test_parallel_fallback_does_not_repeat_python_layer_warningr'!s 388$38:$ C n $DS{+H%t[EFH!bjjoOKHa  | ## #&(+08 9WC 9 2>   9s0CrrrSrcddjdtdDz}|dz }|j|t|}t t ||fi|\}}t t jtj|fi|\}} |gk(sJtj||t|dk(sJ|djtusJdt|dj vsJy) Nrr.c3NK|]}|d|dzdjywrrrs r,rz=test_multibyte_sep_or_quotechar_warns_once..Crr"rz multibyte.csvrGrr)rrrcrrr r#r/rrrarbrrr rr) rKrLr0rr(rr5rrfrs r,*test_multibyte_sep_or_quotechar_warns_oncer*=s 388#RuSz#RR RC o %DS{+H%t[4:FH!bjjoHHKHa r>>&(+ x=A   A;  = 00 0 0C 8K8K4L LL Lr.big922337203685477580718446744073709551616z/WASM stays serial, so the chunks never disagreecTtjdd|djdzzdz}|dz }|j|t |}t ||d}t jtj|d}|dgk(sJtj||y) Nrxcol1 r~rs1.5 z huge_int.csvrSr) rrTrrcrr r#r/rrrarb)rKrLr+rr(rr5rfs r,3test_parallel_pyarrow_huge_int_chunk_matches_serialr0Vs  " Rj'')C/ /( :C n $DS{+H "4I NF{{2::c?)DH  | ## #&(+r.z.WASM stays serial, so nothing infers per chunkc,tjddddzz}|dz }|j|t|}t ||d}t j tj|d}|dgk(sJtj||y)Nrxr/s9223372036854775807 rzhuge_int_only.csvrSr) rrTrcrr r#r/rrrarb)rKrLrr(rr5rfs r,4test_parallel_pyarrow_huge_int_column_stays_parallelr2ks   " .4 4C ) )DS{+H "4I NF{{2::c?)DH x  &(+r.z.WASM stays serial, so no sample line is parsedc:tjdddjdtdDz}|dz }|j |t ||d}t j|tjtj|dy)Nrxscol1 18446744073709551616 r.c3BK|]}|djyw)z.5 Nrrs r,rzCtest_parallel_huge_int_first_line_matches_serial..s$5 !1#T 5srzhuge_first.csvrS) rrTrrrcr rarbr#r/rr)rKrLrr(r5s r,0test_parallel_huge_int_first_line_matches_serialr5}s   " )CHH5%*3Z5- C & &DS "4I NF&"++bjjoY"WXr.r )r4min_chunk_rowsc gtjfd}|jtd||jtdd|jtd|||jtd|tjd|5tj |fi|dddrd Sd S#1swYxYw) a`Return the chunk count ``read_csv`` actually plans for *path*. Reads the split the planner produces rather than recomputing it, so a test cannot agree with a rule the code no longer follows. *chunk_bytes* and *min_chunk_rows* stand in for the module constants so a test-sized fixture can still exercise the one bound it is about. cX|||g|}jt|dz |Sr)rr)rrrrrrrs r,rz!_planned_chunk_count..spys08XzADA CL1$%r.rrFrG_PARALLEL_CHUNK_BYTESN_PARALLEL_MIN_CHUNK_ROWSrr)rIrrHr#rr/) r(rL chunk_bytesr4r6r0rrrs @@r,_planned_chunk_countr<s D00H ".s5!!g5rc3"K|]}ywr r)rrfields r,rz_write_grid..s0Q50s r~)rrrcr)r(r)n_colsr@r:rs ` r, _write_gridrBs` XX5uV}5 5F ((0%-0 0C ftmRj6&99AACDr.cT|dz }|dz }t|dddt|ddd|jjd|jjzkDsJt||dd t||dd k(sJt||d d }t||d d }||kDsJy) Nzlean.csvzfat.csvrr 7r@ 7777777777rGr;r6)rBstatst_sizer<)rKrLleanfatlean_nfat_ns r,+test_parallel_chunk_count_follows_file_sizerPs j D Y Cfas+VQl3 88:  DIIK$7$7 7 77 7  kwq  c;GTU V WW W"$ VW XF ktTU VE 6>>r.cv|dz }t|ddtdD]}t||ddd}|dk\rJy) NrOrrGr$rrr;r4r6rq)rBrr<)rKrLr(rrs r,)test_parallel_read_many_chunks_per_workerrSsR j Dfa 2Y ' +21 4  r.cH|dz }|dz }t|ddt|ddt||dd }t||dd }||ksJ|jj|jjkDsJtj dz}|| dz dzk(sJy) Nz narrow.csvrir irtrIrGrH)rBr<rJrKrI_PARALLEL_MAX_COLUMN_PIECES)rKrLnarrowwidenarrow_nwide_nbudgets r,2test_parallel_chunk_count_bounded_by_column_piecesr[s $F j D#eS!# aH"$ VW XF H   99;  !6!6 66 6  1 1S 8F w!|_q( (( (r.c`|dz }t|ddtt|dd}tt|d|d}t j |Dcgc] \}}||z  }}}|d|dd z ksJt |d dt |dd ksJ|d|k(sJ|d|jjk(sJtt|d|}t j |Dcgc] \}}||z  }}}t|t|d zksJycc}}wcc}}w) Ngrid.csvrr rrrrrrir?) rBrrr itertoolspairwiserurJrKmaxmin) rKr(rrstartendsizesequal equal_sizess r,;test_parallel_chunk_sizes_taper_towards_the_end_of_the_filerhs5 j Dfa (T1qIJ&s4y"jANG+4+=+=g+F GZUCS5[ GE G 9uQx!| ## # uRSz?Sr^ ++ + 1: ## # 2;$))+-- -- - %SYJ ?E1:1C1CE1JK:5#3;KKK { c+.5 55 5 HLs D$5D*rc|dz }t|ddtt|dd}tt|||d}t j |Dcgc] \}}||z  }}}t |t|dzksJycc}}w)Nr]rr rrrr^)rBrrrr_r`rarb)rKrr(rrrcrdres r,7test_parallel_chunk_sizes_do_not_taper_within_one_roundrj s j Dfa (T1qIJ&s4y(JRSTG+4+=+=g+F GZUCS5[ GE G u:E T) )) ) HsB r4)r rrrcp|dz }t|ddt||d|d}||zdk(sJ||dzkDsJy) Nr]rr rIrGrRrrvrBr<)rKrLr4r(rs r,5test_parallel_chunk_count_is_a_whole_number_of_roundsrmsY j Dfa # ktyQRH i 1 $$ $ i!m ## #r.rArc|dz }t|d|t||d|d}t||d|d}||dzk(sJ||kDsJy)Nr]rrGrGrRrIrvrl)rKrLrAr4r(without with_terms r,5test_parallel_chunk_count_never_below_the_worker_rulerr-sj j Dff%" kw)TUG% ktyQRI i!m ## # w  r.c|dz }t|dd|jjtjksJt ||dd}t ||tjd}||k(sJt ||dd|kDsJy)Nz smallish.csvrr rGrGrHi)rBrJrKrIr9r<)rKrLsmallrprqs r,8test_parallel_chunk_count_unchanged_below_the_block_sizeruEs ~ %Eua ::<  ("@"@ @@ @" {G% 22 I    UKSQRS   r.c|dz }t|dddt||d}t||d}||k(sJ|jjdzd|zkDsJy) Nz row_poor.csvrr xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxrErG)r;rI)rBr<rJrK)rKrLr(rprqs r,0test_parallel_chunk_count_respects_the_row_floorrx`si n $Dc2Y/"4'JG$T;DII    99;  $ &i 77 7r.c ,tjjd}tdDcic]}d||j dd|}}|j tdDcic]}d||j|c}|j tdDcic]}d||j gd |c}|j dd |jt|d <tj|Scc}wcc}wcc}w) NrrrirqrrZr/)id1id22id333rb0) r r!r"rr%r?r&astypeboolr#r$)r)r*kdatas r,_blocked_framerts ))   "CAFq<=!A3eT77D KKeAh?1QC#**V,,?@KKMRSTXV1QC#**5F*C CVa077=DJ << @VsD&D ! Dthreadsrcz|dz }tj|d|jtdd|jtddt j d|5t j|}dddt j|d }tj|y#1swY7xYw) Nz blocked.csvFrrFrG _BLOCK_BYTESrrwrT) rr'rHrI_parsersr#rr/rarb)rKrLrr(r5rfs r,1test_row_blocked_conversion_matches_python_enginers m #DD.";;t9=>"5w?  -w 7<T;<&(+ >><rs  #   , 14 !$z<z.*  yk" uyk*+ *+ ,,$D!MN,O,0  Q,& 6 24 , < PD!MN ,O ,D!MN O "D!MN #w(<#s*DE,O,6D!MN O " $,",$,( -$F8ff-=">?D!UV,W@,6D!NOP4D!IJK0 GD!IJK8D!WXY8I2[$$7%#GHMIM0!68N OPD!RS,TQ,&D!QR,S,"D!QR YS Y,234""8ED!QRS*D!QR  S  D!QR)S),60aV, *- *D!QRl3$4S$&D!QRAr7+q"g./,S*D!QRS4D!QR 8S 8& QF+ ,, ,QF+(,(&QF+,,,.QF+,,,r.