""" Module contains tools for processing files into DataFrames or other objects GH#48849 provides a convenient way of deprecating keyword arguments """ from __future__ import annotations import codecs from collections import ( abc, defaultdict, ) from concurrent.futures import ThreadPoolExecutor import contextlib import csv import io import mmap import os import queue import sys import threading from typing import ( IO, TYPE_CHECKING, Any, Generic, Literal, Self, TypedDict, Unpack, cast, overload, ) import warnings import numpy as np from pandas._config import get_option from pandas._libs import lib from pandas._libs.parsers import STR_NA_VALUES from pandas.compat._cpu import available_cpu_count from pandas.errors import ( AbstractMethodError, EmptyDataError, Pandas4Warning, ParserError, ParserWarning, ) from pandas.util._decorators import ( set_module, ) from pandas.util._exceptions import find_stack_level from pandas.util._validators import ( check_dtype_backend, validate_bool_kwarg, ) from pandas.core.dtypes.common import ( is_float, is_integer, is_list_like, pandas_dtype, ) from pandas.core.dtypes.dtypes import ArrowDtype from pandas.core.dtypes.inference import is_file_like from pandas import Series from pandas.core.arrays import ( DatetimeArray, TimedeltaArray, ) from pandas.core.frame import DataFrame from pandas.core.indexes.api import ( RangeIndex, ensure_index, ) from pandas.core.internals.api import _make_block from pandas.core.internals.managers import BlockManager from pandas.io.common import ( IOHandles, get_handle, infer_compression, stringify_path, validate_header_arg, ) from pandas.io.parsers.arrow_parser_wrapper import ArrowParserWrapper from pandas.io.parsers.base_parser import ( ParserBase, is_index_col, parser_defaults, ) from pandas.io.parsers.c_parser_wrapper import ( CParserWrapper, _concatenate_chunks, ) from pandas.io.parsers.python_parser import ( FixedWidthFieldParser, PythonParser, ) if TYPE_CHECKING: from collections.abc import ( Callable, Hashable, Iterable, Mapping, Sequence, ) from types import TracebackType from pandas._typing import ( CompressionOptions, CSVEngine, DtypeArg, DtypeBackend, FilePath, HashableT, IndexLabel, ReadCsvBuffer, StorageOptions, UsecolsArgType, ) class _read_shared(TypedDict, Generic[HashableT], total=False): # annotations shared between read_csv/fwf/table's overloads # NOTE: Keep in sync with the annotations of the implementation sep: str | lib.NoDefault | None delimiter: str | lib.NoDefault | None header: int | Sequence[int] | Literal["infer"] | None names: Sequence[Hashable] | lib.NoDefault | None index_col: IndexLabel | Literal[False] | None usecols: UsecolsArgType dtype: DtypeArg | None engine: CSVEngine | None converters: Mapping[HashableT, Callable] | None true_values: list | None false_values: list | None skipinitialspace: bool skiprows: list[int] | int | Callable[[Hashable], bool] | None skipfooter: int nrows: int | None na_values: ( Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None ) keep_default_na: bool na_filter: bool skip_blank_lines: bool parse_dates: bool | Sequence[Hashable] | None date_format: str | dict[Hashable, str] | None dayfirst: bool cache_dates: bool compression: CompressionOptions thousands: str | None decimal: str lineterminator: str | None quotechar: str quoting: int doublequote: bool escapechar: str | None comment: str | None encoding: str | None encoding_errors: str | None dialect: str | csv.Dialect | type[csv.Dialect] | None on_bad_lines: str low_memory: bool memory_map: bool float_precision: Literal["high", "legacy", "round_trip"] | None storage_options: StorageOptions | None dtype_backend: DtypeBackend | lib.NoDefault else: _read_shared = dict class _C_Parser_Defaults(TypedDict): na_filter: Literal[True] low_memory: Literal[True] memory_map: Literal[False] float_precision: None _c_parser_defaults: _C_Parser_Defaults = { "na_filter": True, "low_memory": True, "memory_map": False, "float_precision": None, } class _Fwf_Defaults(TypedDict): colspecs: Literal["infer"] infer_nrows: Literal[100] widths: None _fwf_defaults: _Fwf_Defaults = {"colspecs": "infer", "infer_nrows": 100, "widths": None} _c_unsupported = {"skipfooter"} _python_unsupported = {"low_memory", "float_precision"} # Documented as `bool` and consumed for their truthiness, so an unvalidated # non-bool like "False" silently means the opposite (GH#68341). _bool_kwargs = frozenset( { "cache_dates", "dayfirst", "doublequote", "iterator", "keep_default_na", "low_memory", "memory_map", "na_filter", "skip_blank_lines", "skipinitialspace", } ) def _is_bool_like(value: object) -> bool: # Only 0 and 1 stand in for the bools, numpy ints included; a larger int is # truthy but not a bool, see test_bool_kwarg_int_not_zero_or_one (GH#68341) return lib.is_bool(value) or (is_integer(value) and value in (0, 1)) def _validate_bool_kwargs(kwds: Mapping[str, Any]) -> None: # iterate kwds, not the frozenset, so the kwarg named is the first in # signature order rather than whichever one hash randomization picks for kwd, value in kwds.items(): if kwd in _bool_kwargs and not _is_bool_like(value): # always raises; validate_bool_kwarg owns the message validate_bool_kwarg(value, kwd, none_allowed=False) # Minimum file size (bytes) to attempt parallel CSV reading. # Below this threshold the overhead of splitting and threading outweighs the benefit. # Break-even is around 1-2 MB; 5 MB leaves margin for cold-cache reads. _PARALLEL_READ_MIN_BYTES = 5 * 1024 * 1024 # 5 MB # Minimum rows per parallel chunk, bounding how finely a file is split. _PARALLEL_MIN_CHUNK_ROWS = 2000 # Target bytes per parallel chunk. Between pyarrow's 1 MB read block and # DuckDB's 8 MB per-thread unit. _PARALLEL_CHUNK_BYTES = 4 * 1024 * 1024 # 4 MB # Ceiling on the (column x chunk) pieces the byte-driven count may create, # bounding the per-column cost (a GIL-held allocation and dtype inference) # that every chunk repeats however wide the frame is. _PARALLEL_MAX_COLUMN_PIECES = 1800 # Size of the last chunk relative to a full one. Chunks are handed out first # come, first served, so the read ends when the last chunk *started* finishes; # tapering leaves a worker that arrives late something short to take. 1.0 # disables the taper. _PARALLEL_TAPER_RATIO = 0.2 # Ceiling on the *default* parallel-read worker count: parallel CSV reading # sees diminishing returns beyond a handful of workers, and a low default # avoids oversubscribing the machine. mode.max_threads overrides it in either # direction. _MAX_DEFAULT_WORKERS = 6 _pyarrow_unsupported = { "skipfooter", "float_precision", "chunksize", "comment", "nrows", "thousands", "memory_map", "dialect", "quoting", "lineterminator", "converters", "iterator", "dayfirst", "skipinitialspace", "low_memory", "na_filter", } @overload def validate_integer(name: str, val: None, min_val: int = ...) -> None: ... @overload def validate_integer(name: str, val: float, min_val: int = ...) -> int: ... @overload def validate_integer(name: str, val: int | None, min_val: int = ...) -> int | None: ... def validate_integer( name: str, val: int | float | None, min_val: int = 0 ) -> int | None: """ Checks whether the 'name' parameter for parsing is either an integer OR float that can SAFELY be cast to an integer without losing accuracy. Raises a ValueError if that is not the case. Parameters ---------- name : str Parameter name (used for error reporting) val : int or float The value to check min_val : int Minimum allowed value (val < min_val will result in a ValueError) """ if val is None: return val msg = f"'{name:s}' must be an integer >={min_val:d}" if is_float(val): if int(val) != val: raise ValueError(msg) val = int(val) elif not (is_integer(val) and val >= min_val): raise ValueError(msg) return int(val) def _validate_names(names: Sequence[Hashable] | None) -> None: """ Raise ValueError if the `names` parameter contains duplicates or has an invalid data type. Parameters ---------- names : array-like or None An array containing a list of the names used for the output DataFrame. Raises ------ ValueError If names are not unique or are not ordered (e.g. set). """ if names is not None: if len(names) != len(set(names)): raise ValueError("Duplicate names are not allowed.") if not ( is_list_like(names, allow_sets=False) or isinstance(names, abc.KeysView) ): raise ValueError("Names should be an ordered collection.") def _read( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], kwds ) -> DataFrame | TextFileReader: """Generic reader of line files.""" # before the `iterator` peek below, which reads it for truthiness _validate_bool_kwargs(kwds) # if we pass a date_format and parse_dates=False, we should not parse the # dates GH#44366 if kwds.get("parse_dates", None) is None: if kwds.get("date_format", None) is None: kwds["parse_dates"] = False else: kwds["parse_dates"] = True # Extract some of the arguments (pass chunksize on). iterator = kwds.get("iterator", False) chunksize = kwds.get("chunksize", None) # Check type of encoding_errors errors = kwds.get("encoding_errors", "strict") if not isinstance(errors, str): raise ValueError( f"encoding_errors must be a string, got {type(errors).__name__}" ) if kwds.get("engine") == "pyarrow": if iterator: raise ValueError( "The 'iterator' option is not supported with the 'pyarrow' engine" ) if chunksize is not None: raise ValueError( "The 'chunksize' option is not supported with the 'pyarrow' engine" ) else: chunksize = validate_integer("chunksize", chunksize, 1) nrows = kwds.get("nrows", None) # Check for duplicates in names. _validate_names(kwds.get("names", None)) if kwds.get("float_precision") is not None: warnings.warn( "The 'float_precision' argument is deprecated. " "Use the default float precision instead.", Pandas4Warning, stacklevel=find_stack_level(), ) # For large local uncompressed files with the C engine, attempt parallel reading. # Each worker gets its own view of the mmapped file and its own TextReader so # the GIL-free tokenisation and type-conversion code in parsers.pyx runs truly # in parallel. if not iterator and chunksize is None and nrows is None: _n_workers = _default_n_workers() if _n_workers > 1 and _can_parallelize_csv(filepath_or_buffer, kwds): _filepath = stringify_path(filepath_or_buffer) assert isinstance(_filepath, str) # guaranteed by _can_parallelize_csv try: result = _read_csv_parallel(_filepath, kwds, _n_workers) except (ParserError, UnicodeDecodeError, OverflowError): # e.g. the one-line sample that infers the column names ends # inside a quoted field (GH#66259). The serial path below # handles anything the parallel path cannot -- and raises in # turn if it too fails. # Other exceptions from outside the chunk reads propagate: they # signal a parallel-path bug, not ineligible input. result = None if result is not None: return result # Create the parser. parser = TextFileReader(filepath_or_buffer, **kwds) if chunksize or iterator: return parser with parser: return parser.read(nrows) def _default_n_workers() -> int: """ Default worker count for a parallel ``read_csv``. ``mode.max_threads`` wins whenever it is set (except on Emscripten, which cannot spawn threads at all). Otherwise it is the smallest of the machine's logical CPU count, ``_MAX_DEFAULT_WORKERS``, and the CPUs actually available to the process (CPU affinity / cgroup limits) -- so that an embedded or containerised pandas does not oversubscribe its allocation. """ max_threads = get_option("mode.max_threads") if sys.platform == "emscripten": # WASM cannot spawn threads, regardless of mode.max_threads. return 1 if max_threads is not None: return max_threads n_workers = min(os.cpu_count() or 1, _MAX_DEFAULT_WORKERS) # os.cpu_count() counts the machine's CPUs, not the ones this process may # use, so it alone would put _MAX_DEFAULT_WORKERS parse threads on a # single-CPU container. available = available_cpu_count() if available is not None: n_workers = min(n_workers, available) return n_workers def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool: """ Return True when a ``read_csv`` call is eligible for parallel execution. Parallel reading works by splitting the file's data section into byte-range chunks at newline boundaries and processing them concurrently with threads. The following conditions must all hold: * *filepath_or_buffer* is a local file path (not a URL or file object). * The file is not compressed. * The C engine is selected (``engine='c'``, the default). * The call is not in iterator / chunked mode. * The entire file is being read (no ``nrows`` limit). * ``skiprows`` is a plain integer (or absent) - list/callable forms require tracking absolute line numbers across chunks. * ``header`` is a single integer or ``None`` - multi-level headers complicate the preamble boundary. * ``index_col`` is ``None`` or ``False`` - a column-based index would need its name propagated to non-first chunks. * ``usecols`` is ``None`` - column selection changes the mapping between raw column positions and names in non-first chunks. * The separator is a single ASCII character or ``r"\\s+"``, and ``quotechar`` is a single ASCII character - anything else forces the python engine inside ``TextFileReader``. * The encoding is ``utf-8`` / ``utf-8-sig`` - chunk workers feed raw file bytes to the C tokenizer, which decodes words as UTF-8. ``ascii`` is excluded so a non-ASCII byte still raises rather than being masked. * ``comment`` is ``None`` and the preamble has no blank lines - full-line comments and blank lines shift where pandas locates the header relative to the physical line count used by :func:`_find_data_start_offset`. * ``escapechar`` is ``None`` and ``dialect`` is not used - an escaped literal newline would be split mid-field. * ``parse_dates`` is unset - datetime format inference is per-chunk and may disagree with the serial whole-column inference. * ``low_memory`` is truthy - a falsy ``low_memory`` guarantees whole-file type inference, which per-chunk parallel reading cannot honour (``low_memory=True`` already documents per-chunk inference divergence). * ``storage_options`` is ``None`` - it raises for local paths in the serial path, and that error must not be masked. * ``on_bad_lines`` is not ``"warn"`` - chunk workers would report chunk-relative (i.e. wrong) line numbers. * Both the file and its data section (i.e. excluding the header preamble) are at least ``_PARALLEL_READ_MIN_BYTES`` bytes large. Note that a chunk boundary landing on a newline embedded inside a quoted field leaves that chunk's parser inside an open quote at EOF, so the chunk fails and the read falls back to serial - it does not corrupt data silently. """ # Must be a local file path, not a URL or file-like object. if is_file_like(filepath_or_buffer): return False filepath = stringify_path(filepath_or_buffer) if not isinstance(filepath, str): return False if "://" in filepath: return False # storage_options on a local path raises ValueError in get_handle; the # parallel path strips it and would mask that error. if kwds.get("storage_options") is not None: return False # Uncompressed files only (splitting is only meaningful for raw bytes). compression = kwds.get("compression", "infer") if isinstance(compression, dict): compression = compression.get("method") if compression is not None: try: if infer_compression(filepath, compression) is not None: return False except ValueError: # unrecognized compression argument; let the serial path raise return False # Iterator / chunked mode: caller handles chunking themselves. if kwds.get("iterator", False) or kwds.get("chunksize") is not None: return False # Only the C engine benefits; pyarrow has its own parallelism. # engine=None means "use the default", which is "c". if kwds.get("engine") not in ("c", None): return False # nrows: we'd need to distribute the row budget across chunks - skip for now. if kwds.get("nrows") is not None: return False # The chunk splitter scans for raw \n bytes. A custom lineterminator # (which the C engine does honour) would misalign chunk boundaries. lineterminator = kwds.get("lineterminator") if lineterminator is not None and lineterminator != "\n": return False # Chunk workers feed raw file bytes to the C tokenizer, which decodes # words as UTF-8. The serial path instead transcodes through a # TextIOWrapper, so anything that is not already valid UTF-8 (latin-1, # cp1252, UTF-16/32, ...) must take the serial path. "ascii" is excluded # for the same reason: the workers would decode a non-ASCII byte as UTF-8 # and succeed, masking the UnicodeDecodeError the serial path raises for # bytes outside the declared encoding (GH#64347). encoding = kwds.get("encoding") if encoding is not None: try: codec_name = codecs.lookup(encoding).name except LookupError: return False if codec_name not in ("utf-8", "utf-8-sig"): return False # Separators that force the python-engine fallback inside TextFileReader # (sep=None sniffing, or multi-char/regex seps other than r"\s+") cannot # use the C-engine buffer-loading fast path. delimiter = kwds.get("delimiter", ",") if delimiter is None or (len(delimiter) > 1 and delimiter != r"\s+"): return False # A separator or quotechar wider than one byte forces the python engine # too, and warns on the way. The eligibility check has to catch these # here: by the time the name-inference read reports the python engine, it # has already raised that warning, which the serial read then raises # again. GH#66259 if len(delimiter) == 1 and ord(delimiter) > 127: return False quotechar = kwds.get("quotechar") if isinstance(quotechar, str) and len(quotechar) == 1 and ord(quotechar) > 127: return False # Full-line comments before/inside the preamble shift the header location # in ways _find_data_start_offset cannot see. if kwds.get("comment") is not None: return False # An escaped literal newline (escapechar followed by "\n") would be split # mid-field. if kwds.get("escapechar") is not None: return False # dialect is merged into the kwds inside TextFileReader, i.e. after this # function runs, so a dialect-specified escapechar would bypass the check # above. Be conservative and take the serial path. if kwds.get("dialect") is not None: return False # Datetime format inference is per-chunk and may disagree with the serial # whole-column inference. if kwds.get("parse_dates"): return False # A falsy low_memory guarantees whole-file type inference, which per-chunk # parallel reading cannot honour (GH#64347). Test truthiness, not identity: # low_memory is not coerced to bool, and the serial path in # CParserWrapper.read also branches on truthiness, so low_memory=0 / # np.False_ must take the serial path too (GH#66327). if not kwds.get("low_memory", True): return False # on_bad_lines="warn" includes line numbers in its warnings; chunk workers # would report chunk-relative (i.e. wrong) ones. if kwds.get("on_bad_lines") == ParserBase.BadLineHandleMethod.BLHM_WARN: return False # Callable / list skiprows require tracking absolute line numbers. skiprows = kwds.get("skiprows") if skiprows is not None and not isinstance(skiprows, (int, np.integer)): return False # Multi-level (list) headers span multiple preamble lines - skip for now. if isinstance(kwds.get("header", "infer"), (list, tuple)): return False # skipfooter: the C engine doesn't support it anyway, but be explicit. if kwds.get("skipfooter", 0) > 0: return False # A column-based index_col would need its name propagated to non-first chunks. index_col = kwds.get("index_col", None) if index_col is not None and index_col is not False: return False # usecols changes the column name ↔ position mapping in non-first chunks. if kwds.get("usecols") is not None: return False # Only bother for files large enough to amortise the threading overhead. try: file_size = os.path.getsize(filepath) if file_size < _PARALLEL_READ_MIN_BYTES: return False except OSError: return False # Check that the *data section* (after header/skiprows preamble) is large # enough to split into at least 2 chunks. Without this, _read_csv_parallel # would have to return None for files whose preamble eats most of the bytes. _header = kwds.get("header", "infer") if _header == "infer": _header = 0 if kwds.get("names") is None else None _skiprows = int(kwds.get("skiprows") or 0) data_start = _find_data_start_offset(filepath, _header, _skiprows) if file_size - data_start < _PARALLEL_READ_MIN_BYTES: return False # Blank lines in the preamble shift where pandas locates the header # relative to the physical line count used by _find_data_start_offset. with open(filepath, "rb") as fd: preamble = fd.read(data_start) if any(not line.rstrip(b"\r") for line in preamble.splitlines()): return False return True def _find_data_start_offset( filepath: str, header: int | None, skiprows: int, ) -> int: """ Return the byte offset of the first data row in *filepath*. Reads ``int(skiprows) + (header + 1 if header is not None else 0)`` physical lines from the start of the file and returns the position after the last preamble byte. This is the byte from which each non-first chunk must begin. """ if header is None: header_lines = 0 else: header_lines = int(header) + 1 # header=0 → 1 header line preamble_lines = int(skiprows) + header_lines with open(filepath, "rb") as fd: for _ in range(preamble_lines): if not fd.readline(): break return fd.tell() def _chunk_size_weights(n_chunks: int, n_tapered: int) -> list[float]: """ Relative sizes for *n_chunks* chunks whose last *n_tapered* shrink. The chunks before the tail are full size; across the tail the size falls linearly to ``_PARALLEL_TAPER_RATIO`` of one, so the final chunk is the smallest piece of work in the file. """ weights = [1.0] * n_chunks n_tapered = min(n_tapered, n_chunks) for position in range(n_tapered): share = (position + 1) / n_tapered weights[n_chunks - n_tapered + position] = ( 1.0 - (1.0 - _PARALLEL_TAPER_RATIO) * share ) return weights def _find_chunk_byte_offsets( filepath: str, n_chunks: int, data_start: int, n_workers: int = 0, ) -> list[int]: """ Compute byte offsets that partition the data portion of *filepath* into *n_chunks* pieces aligned to newline boundaries. The pieces are equal-sized, except that when *n_workers* is given and the chunks take more than one round the last ``2 * n_workers`` taper down: the read finishes when the last chunk a worker picked up does, so ending on small chunks costs less than ending on a full one. Within a single round every chunk runs at once, so the makespan is the largest of them and any taper would only inflate it. Returns a list of ``n + 1`` offsets where the byte range ``[offsets[i], offsets[i+1])`` defines chunk *i*. """ file_size = os.path.getsize(filepath) data_size = file_size - data_start offsets = [data_start] if n_chunks <= 1 or data_size <= 0: offsets.append(file_size) return offsets n_tapered = 2 * n_workers if n_chunks > n_workers else 0 weights = _chunk_size_weights(n_chunks, n_tapered) total_weight = sum(weights) running_weight = 0.0 with open(filepath, "rb") as fd: for i in range(1, n_chunks): running_weight += weights[i - 1] target = data_start + int(data_size * running_weight / total_weight) if target >= file_size: break fd.seek(target) fd.readline() # advance past the partial line at the split point pos = fd.tell() if pos >= file_size: break if pos == offsets[-1]: # This target fell inside the line the previous boundary # already ended; skip it rather than abandoning the boundaries # after it, so one long line cannot collapse the whole split. continue offsets.append(pos) offsets.append(file_size) return offsets def _raise_collected(warning_sink: list[tuple[str, type[Warning]]]) -> None: """Raise each distinct warning the parallel read's parsers collected.""" for warn_msg, warn_category in dict.fromkeys(warning_sink): warnings.warn(warn_msg, warn_category, stacklevel=find_stack_level()) def _read_csv_parallel( filepath: str, kwds: dict, n_workers: int, ) -> DataFrame | None: """ Read a large CSV file in parallel using *n_workers* threads. Every read this makes - the one-line name inference and each chunk - raises its own copy of the same ``ParserWarning``, and a worker raising one lands the stacklevel walk in threading internals. So the reads collect their warnings into a sink (see :attr:`TextReader.warning_sink`) and each distinct one is raised once here, from the caller's frame - except when the caller goes on to read the file serially, since that read raises them itself. GH#66259 The file's data section (everything after the header / skiprows preamble) is split into byte-range chunks aligned to newline boundaries, sized from *n_workers* and from the file's own rows, bytes and column count. Each chunk is parsed by an independent :class:`TextFileReader` / C-engine instance. Because the hot paths in ``pandas/_libs/parsers.pyx`` (tokenisation, int/float/bool conversion) are wrapped in ``with nogil:`` blocks, threads achieve real CPU-level parallelism. The caller must ensure the file is eligible via :func:`_can_parallelize_csv` before calling this function. Returns ``None`` when parallel reading turns out not to be applicable after all (the data section cannot be split, the engine falls back to python, the one-line sample used to infer column names has no columns, per-chunk dtype inference disagrees in a way that would not match the serial result, or any chunk fails to parse); the caller then reads serially. """ warning_sink: list[tuple[str, type[Warning]]] = [] try: result = _read_csv_chunks(filepath, kwds, n_workers, warning_sink) except (ParserError, UnicodeDecodeError, OverflowError): # The caller answers these with a serial read of the whole file. raise except Exception: # Nothing re-reads the file after this, so it is here or nowhere. This # over-warns when a serial read would have died before reaching the # warning's column; losing the warning otherwise is the worse trade. _raise_collected(warning_sink) raise if result is not None: _raise_collected(warning_sink) return result def _read_csv_chunks( filepath: str, kwds: dict, n_workers: int, warning_sink: list[tuple[str, type[Warning]]], ) -> DataFrame | None: """ Body of :func:`_read_csv_parallel`; see there for what the arguments mean and when the result is ``None``. Appends the ``ParserWarning``\\s its parsers would have raised to *warning_sink* instead of raising them. """ # Resolve the effective header value (mirrors TextFileReader.__init__). header = kwds.get("header", "infer") if header == "infer": header = 0 if kwds.get("names") is None else None skiprows = int(kwds.get("skiprows") or 0) # Byte offset at which real data rows begin. data_start = _find_data_start_offset(filepath, header, skiprows) # Oversubscribe the workers so one slow chunk cannot strand a core. Take # the median of sampled line lengths - a single probe lets one atypical # line skew the estimate. The count is raised to follow the file's size # once the column count is known, below. data_size = os.path.getsize(filepath) - data_start line_lens = [] with open(filepath, "rb") as fh: for probe in range(9): fh.seek(data_start + data_size * probe // 9) fh.readline(1 << 20) # advance past the partial line at the probe line = fh.readline(1 << 20) if line: line_lens.append(len(line)) line_lens.sort() bytes_per_row = line_lens[len(line_lens) // 2] if line_lens else data_size est_rows = data_size // bytes_per_row n_target = min(n_workers * 3, est_rows // _PARALLEL_MIN_CHUNK_ROWS) if n_target < 2: # Too few rows to split, unless each half is big enough to amortise # the per-column costs anyway. if data_size < 2 * _PARALLEL_READ_MIN_BYTES: return None n_target = 2 # ------------------------------------------------------------------ # Infer column names from the preamble + one data line (very fast). # ------------------------------------------------------------------ base_kwds: dict = { **kwds, "compression": None, "memory_map": False, "storage_options": None, } with open(filepath, "rb") as fd: preamble = fd.read(data_start) first_line = fd.readline() # The tokenizer also ends a line at a bare \r, which readline does not, so # data_start could be misplaced without the row count check below noticing. if b"\r" in (preamble + first_line).replace(b"\r\n", b""): return None # Only the column names and the row count are kept, so the sample is parsed # as strings: a value that converts on its own line but not for the whole # column (an int above 2**63 under dtype_backend="pyarrow", say) would # otherwise raise here for a file the serial path reads fine. GH#66259 name_buf = io.BytesIO(preamble + first_line) name_kwds = { **base_kwds, "dtype": str, "converters": None, "dtype_backend": lib.no_default, } try: name_reader = TextFileReader(name_buf, **name_kwds) except EmptyDataError: # The one data line we sliced off is blank (e.g. header=None on a file # whose first physical line is empty), so the name-inference read sees # no columns. The serial path skips the blank line and reads the file # fine, so fall back rather than propagate. GH#66259 return None if not isinstance(name_reader._engine, CParserWrapper): # TextFileReader fell back to the python engine for a reason the # eligibility checks did not anticipate; load_buffer needs the C engine. name_reader.close() return None name_reader._engine._warning_sink = warning_sink name_reader._engine._reader.warning_sink = warning_sink if name_reader._engine._reader.leading_cols: # Data rows have more fields than the header (implicit index). The # chunk workers would fail on the extra field; bail out up front. name_reader.close() return None assert name_reader._engine.orig_names is not None col_names: list = list(name_reader._engine.orig_names) # name_buf holds the preamble plus exactly one data line. Parsing it must # yield exactly one row; anything else means the preamble's physical line # count disagrees with its logical row count (e.g. a quoted embedded # newline in the header), so data_start is wrong. n_first_rows = len(name_reader.read()) name_reader.close() if n_first_rows != 1: return None # A dict ``dtype`` is applied per raw header name: when a name is repeated, # the serial path assigns that dtype to every de-duplicated column (``a`` # and ``a.1``), but the workers receive the already-de-duplicated # ``col_names`` and would match only ``a``. ``col_names`` no longer shows # the duplication, so recover the raw header names (parse the header line # as data) and fall back to serial when they collide. GH#64347 if header is not None and isinstance(kwds.get("dtype"), dict): header_line = preamble.splitlines(keepends=True)[-1] raw_reader = TextFileReader( io.BytesIO(header_line), **{ **base_kwds, "header": None, "names": None, "usecols": None, "index_col": False, "dtype": str, "converters": None, "skiprows": None, "na_filter": False, }, ) try: raw_names = list(raw_reader.read().iloc[0]) finally: raw_reader.close() if len(set(raw_names)) != len(raw_names): return None # n_target so far scales with the worker count, which says nothing about # the file: a 128 MB file gets the same split as one just over the size # gate. Raise it to follow the bytes, bounded by the piece budget and by # the row floor, since a file can be byte-rich and row-poor. Only ever # raised, so no file comes out coarser than before. size_target = min( data_size // _PARALLEL_CHUNK_BYTES, _PARALLEL_MAX_COLUMN_PIECES // max(len(col_names), 1), est_rows // _PARALLEL_MIN_CHUNK_ROWS, ) # A count that is not a multiple of the worker count spends its last round # mostly idle: 31 chunks over 6 workers is 5.17 rounds' work that takes 6. # Round up, and only this term, so neither the worker-derived floor nor a # file below one block moves. if size_target > n_workers: size_target = -(-size_target // n_workers) * n_workers n_target = max(n_target, size_target) offsets = _find_chunk_byte_offsets(filepath, n_target, data_start, n_workers) n_chunks = len(offsets) - 1 if n_chunks < 2: # e.g. a data section with no interior newlines (one giant line) return None # Spare threads would only spin up to find the queue empty. n_workers = min(n_workers, n_chunks) # ------------------------------------------------------------------ # Dispatch all chunks in parallel. Each worker gets a zero-copy # memoryview slice of the mmapped file and calls load_buffer() so # tokenisation is fully GIL-free. # ------------------------------------------------------------------ chunk_kwds: dict = { **base_kwds, "header": None, "names": col_names, "skiprows": None, # A worker's byte slice already bounds peak memory, and skipping # low_memory avoids a per-worker GIL-held concatenate. "low_memory": False, } # Each thread reuses one parser to drain the queue, so no worker stalls # the gather on a straggler and no chunk pays parser construction. chunk_queue: queue.SimpleQueue = queue.SimpleQueue() for chunk_idx in range(n_chunks): chunk_queue.put(chunk_idx) results: list = [None] * n_chunks chunk_failed = threading.Event() workers_readers: list = [] def _worker(mv: memoryview) -> None: reader = TextFileReader(io.BytesIO(b""), **chunk_kwds) assert isinstance(reader._engine, CParserWrapper) # Buffers are reused across the queued chunks and freed at close; # trimming between chunks would stall other workers on reallocs. reader._engine._reader.trim_after_read = False # Hand back string columns as raw pending handles: the gather below # combines each column's chunks into one ExtensionArray, so the # GIL-held pyarrow wrap happens once per column rather than once per # chunk in every worker. reader._engine.wrap_deferred = False # Row-blocked conversion of numeric columns pays off once enough # workers contend for memory bandwidth (see TextReader.block_workers). reader._engine._reader.block_workers = n_workers reader._engine._warning_sink = warning_sink reader._engine._reader.warning_sink = warning_sink workers_readers.append(reader) while not chunk_failed.is_set(): try: chunk_idx = chunk_queue.get_nowait() except queue.Empty: return data = mv[offsets[chunk_idx] : offsets[chunk_idx + 1]] # The serial C parser strips a UTF-8 BOM at byte 0 of the file # only; strip_bom mirrors that for the chunk starting there. reader._engine._reader.load_buffer(data, strip_bom=offsets[chunk_idx] == 0) # the reader holds its own reference for the parse duration del data # On a reused parser a zero-row chunk would raise StopIteration # and close the reader; reset so it returns empty meta instead. reader._engine._first_chunk = True try: # Raw column arrays, not DataFrames: per-chunk frame assembly # holds the GIL and the concat would redo the block machinery. _, chunk_columns, col_dict = reader._engine.read() except Exception: chunk_failed.set() return results[chunk_idx] = (chunk_columns, col_dict) def _copy_pieces(jobs: list[tuple[np.ndarray, int, np.ndarray]]) -> None: # ndarray slice assignment releases the GIL for the memcpy, so the # gather parallelises across the pool. for dst, pos, piece in jobs: dst[pos : pos + len(piece)] = piece def _piece_ndarray(piece) -> np.ndarray | None: # The ndarray behind pieces whose final block is ndarray-backed: # plain ndarrays plus tz-naive datetime64/timedelta64. if isinstance(piece, np.ndarray): if piece.dtype == object: # Object pieces take the leftover path below, which runs # the same constructor inference as a serial read. return None return piece if isinstance(piece, (DatetimeArray, TimedeltaArray)) and isinstance( piece.dtype, np.dtype ): return piece._ndarray return None dtype_arg = kwds.get("dtype") # Mirror TextFileReader.read: a dict or str/object dtype needs a # Series-per-column wrap, so it skips direct block assembly. needs_series_wrap = isinstance(dtype_arg, dict) or ( dtype_arg is not None and pandas_dtype(dtype_arg) in (np.str_, np.object_) ) block_specs: list[tuple] = [] leftover_pos: list[int] = [] col_list: list = [] chunk_dicts: list[dict] = [] columns: list[Hashable] = [] total = 0 readers_closing = False # mmap the file once and pass zero-copy memoryview slices to each thread # instead of having each thread open/seek/read its own copy. mmap dups # the file descriptor, so the file object can be closed right away. Map it # last, so that every path out of here runs the close below. GH#66259 with open(filepath, "rb") as fh: mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) try: with ( contextlib.ExitStack() as views, ThreadPoolExecutor(max_workers=n_workers) as pool, ): # One view per worker: slicing a shared memoryview from several # threads races on free-threaded builds (GH#71418). worker_views = [ views.enter_context(memoryview(mm)) for _ in range(n_workers) ] for fut in [pool.submit(_worker, mv) for mv in worker_views]: fut.result() if chunk_failed.is_set(): # Let the serial read raise: a chunk's error can be an artifact # of the split (a boundary inside a quoted newline), and even a # genuine one differs from serial in which error comes first # and in the row positions it reports. return None # A column of only NA tokens and ints too large for int64 converts # to no numeric dtype, and is then emitted with its NA tokens left # as literal strings (GH#14983). Which chunks hit that depends on # how the file was split, and the resulting dtype is str either # way, so the reconciliation below cannot see the difference -- the # values would just silently disagree with a serial read. GH#66259 if any( reader._engine._reader.na_left_literal for reader in workers_readers ): return None # Freeing parser buffers costs a few ms of madvise; run it on the # pool, overlapped with the gather copies below. close_futures = [pool.submit(reader.close) for reader in workers_readers] readers_closing = True # Zero-row chunks carry empty-meta object dtypes that would # needlessly trip the dtype reconciliation below. chunk_results = [ res for res in cast("list[tuple]", results) if res[1] and len(next(iter(res[1].values()))) > 0 ] if not chunk_results: # every chunk parsed to zero rows (e.g. all blank lines); # let the serial path build the empty frame return None columns = list(chunk_results[0][0]) chunk_dicts = [col_dict for _, col_dict in chunk_results] col_list = list(chunk_dicts[0]) if col_list: total = sum(len(chunk_dict[col_list[0]]) for chunk_dict in chunk_dicts) # Per-chunk dtype inference can disagree with whole-file inference # (a lone non-numeric row makes only its own chunk object). Mixed # signed-int/float chunks gather to the float64 serial would give; # anything else must be re-read serially. ArrowDtype is excluded # because pyarrow widens int to double with a *checked* cast, which # raises above 2**53 where the serial whole-file read just infers # double. GH#66259 for name in col_list: chunk_dtypes = {chunk_dict[name].dtype for chunk_dict in chunk_dicts} if len(chunk_dtypes) > 1 and not all( dt.kind in "if" and not isinstance(dt, ArrowDtype) for dt in chunk_dtypes ): return None if not needs_series_wrap: # Gather same-dtype ndarray columns straight into one # preallocated consolidated block per dtype; anything else # falls to _concatenate_chunks below. groups: dict[np.dtype, list[int]] = {} col_nd_pieces: dict[int, list[np.ndarray]] = {} for pos, name in enumerate(col_list): nd_pieces = [ _piece_ndarray(chunk_dict[name]) for chunk_dict in chunk_dicts ] first = nd_pieces[0] if first is not None and all( piece is not None and piece.dtype == first.dtype for piece in nd_pieces ): groups.setdefault(first.dtype, []).append(pos) col_nd_pieces[pos] = cast("list[np.ndarray]", nd_pieces) else: leftover_pos.append(pos) copy_jobs: list[tuple[np.ndarray, int, np.ndarray]] = [] for group_dtype, positions in groups.items(): values2d = np.empty((len(positions), total), dtype=group_dtype) for row, pos in enumerate(positions): offset = 0 for piece in col_nd_pieces[pos]: copy_jobs.append((values2d[row], offset, piece)) offset += len(piece) block_specs.append((values2d, np.array(positions, dtype=np.intp))) # Batch the copies instead of submitting one task per # (column, chunk): on a wide frame that is n_columns * n_chunks # tasks, each moving a sliver of one column, and the dispatch # then costs many times the memcpy it schedules. batch = -(-len(copy_jobs) // (n_workers * 4)) or 1 copy_futures = [ pool.submit(_copy_pieces, copy_jobs[start : start + batch]) for start in range(0, len(copy_jobs), batch) ] for fut in copy_futures: fut.result() for fut in close_futures: fut.result() finally: if not readers_closing: for reader in workers_readers: try: reader.close() except Exception: pass # The closes above drop the chunk slices the readers hold, so the # mapping is normally unmapped right here. suppress: a traceback that # still pins a slice would make close() raise BufferError and mask the # exception on its way out; the mapping then goes when it is collected. with contextlib.suppress(BufferError): mm.close() index = RangeIndex(total) if needs_series_wrap: data = _concatenate_chunks(chunk_dicts, columns, warn_mixed=False) if isinstance(dtype_arg, dict): dtype: defaultdict = defaultdict(lambda: None) dtype.update(dtype_arg) else: dtype = defaultdict(lambda: dtype_arg) new_col_dict = { k: Series( v, index=index, dtype=( dtype[k] if pandas_dtype(dtype[k]) in (np.str_, np.object_) else None ), copy=False, ) for k, v in data.items() } return DataFrame(new_col_dict, columns=columns, index=index, copy=False) if leftover_pos: leftover_names = [col_list[pos] for pos in leftover_pos] leftover_dicts = [ {name: chunk_dict[name] for name in leftover_names} for chunk_dict in chunk_dicts ] gathered = _concatenate_chunks(leftover_dicts, leftover_names, warn_mixed=False) for pos, name in zip(leftover_pos, leftover_names, strict=True): values = gathered[name] if isinstance(values, np.ndarray) and values.dtype == object: # Match the serial constructor's inference for object # columns (object -> str dtype under future.infer_string). values = Series(values, index=index, copy=False)._values if isinstance(values, np.ndarray): # _make_block expects numpy values already in 2D block shape values = values.reshape(1, -1) block_specs.append((values, np.array([pos], dtype=np.intp))) block_objs = [_make_block(values, placement) for values, placement in block_specs] mgr = BlockManager(block_objs, [ensure_index(columns), index]) return DataFrame._from_mgr(mgr, mgr.axes) @overload def read_csv( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: Literal[True], chunksize: int | None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> TextFileReader: ... @overload def read_csv( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: bool = ..., chunksize: int, **kwds: Unpack[_read_shared[HashableT]], ) -> TextFileReader: ... @overload def read_csv( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: Literal[False] = ..., chunksize: None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> DataFrame: ... @overload def read_csv( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: bool = ..., chunksize: int | None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> DataFrame | TextFileReader: ... @set_module("pandas") def read_csv( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, sep: str | lib.NoDefault | None = lib.no_default, delimiter: str | lib.NoDefault | None = lib.no_default, # Column and Index Locations and Names header: int | Sequence[int] | Literal["infer"] | None = "infer", names: Sequence[Hashable] | lib.NoDefault | None = lib.no_default, index_col: IndexLabel | Literal[False] | None = None, usecols: UsecolsArgType = None, # General Parsing Configuration dtype: DtypeArg | None = None, engine: CSVEngine | None = None, converters: Mapping[HashableT, Callable] | None = None, true_values: list | None = None, false_values: list | None = None, skipinitialspace: bool = False, skiprows: list[int] | int | Callable[[Hashable], bool] | None = None, skipfooter: int = 0, nrows: int | None = None, # NA and Missing Data Handling na_values: ( Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None ) = None, keep_default_na: bool = True, na_filter: bool = True, skip_blank_lines: bool = True, # Datetime Handling parse_dates: bool | Sequence[Hashable] | None = None, date_format: str | dict[Hashable, str] | None = None, dayfirst: bool = False, cache_dates: bool = True, # Iteration iterator: bool = False, chunksize: int | None = None, # Quoting, Compression, and File Format compression: CompressionOptions = "infer", thousands: str | None = None, decimal: str = ".", lineterminator: str | None = None, quotechar: str = '"', quoting: int = csv.QUOTE_MINIMAL, doublequote: bool = True, escapechar: str | None = None, comment: str | None = None, encoding: str | None = None, encoding_errors: str | None = "strict", dialect: str | csv.Dialect | type[csv.Dialect] | None = None, # Error Handling on_bad_lines: str = "error", # Internal low_memory: bool = _c_parser_defaults["low_memory"], memory_map: bool = False, float_precision: Literal["high", "legacy", "round_trip"] | None = None, storage_options: StorageOptions | None = None, dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default, ) -> DataFrame | TextFileReader: """ Read a comma-separated values (csv) file into DataFrame. Also supports optionally iterating or breaking of the file into chunks. Additional help can be found in the online docs for `IO Tools `_. Parameters ---------- filepath_or_buffer : str, path object or file-like object Any valid string path is acceptable. The string could be a URL. Valid URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is expected. A local file could be: file://localhost/path/to/table.csv. Certain URL schemes may require additional packages. For example, S3 URLs require the ``s3fs`` library. See :ref:`install.optional_dependencies` for a full list. If you want to pass in a path object, pandas accepts any ``os.PathLike``. By file-like object, we refer to objects with a ``read()`` method that accepts an optional size argument, such as a file handle (e.g. via builtin ``open`` function) or ``StringIO``. sep : str, default ',' Character or regex pattern to treat as the delimiter. ``sep=None`` detects the separator from the first valid row of the file with Python's builtin sniffer tool, ``csv.Sniffer``; it is supported only by the Python parsing engine, which will be used automatically. In addition, separators longer than 1 character and different from ``'\\s+'`` will be interpreted as regular expressions and will force the use of the Python parsing engine. Note that regex delimiters are prone to ignoring quoted data. Regex example: ``'\\r\\t'``. delimiter : str, optional Alias for ``sep``. header : int, Sequence of int, 'infer' or None, default 'infer' Row index or indices in the file to use as DataFrame column labels. Indexing starts at 0 and counts only non-blank, non-commented lines when ``skip_blank_lines=True``, so ``header=0`` denotes the first line of data rather than the first line of the file. Valid arguments are: * ``int``: line index at which column labels are read. * Sequence of ``int``: line indices at which column labels are read and combined into a :class:`~pandas.MultiIndex` on the columns, e.g. ``[0, 1, 3]``. Intervening rows that are not specified will be skipped (e.g. row 2 in this example). * ``None``: no row in the file is interpreted as column labels. Columns are labelled by integer position, or by the values passed to ``names`` when provided. Use this for files with no header row. If the file has a header row that should be overridden by ``names``, pass ``header=0`` instead. * ``'infer'`` (default): equivalent to ``header=0`` when no ``names`` are passed, otherwise equivalent to ``header=None``. When inferred from the file contents, headers are kept distinct from each other by renaming duplicate names with a numeric suffix of the form ``".{count}"`` starting from 1, e.g. ``"foo"`` and ``"foo.1"``. Empty headers are named ``"Unnamed: {i}"`` or `` "Unnamed: {i}_level_{level}"`` in the case of MultiIndex columns. names : Sequence of Hashable, optional Sequence of column labels to apply. If the file contains a header row, then you should explicitly pass ``header=0`` to override the column names. Duplicates in this list are not allowed. index_col : Hashable, Sequence of Hashable or False, optional Column(s) to use as row label(s), denoted either by column labels or column indices. If a sequence of labels or indices is given, :class:`~pandas.MultiIndex` will be formed for the row labels. Note: ``index_col=False`` can be used to force pandas to *not* use the first column as the index, e.g., when you have a malformed file with delimiters at the end of each line. usecols : Sequence of Hashable or Callable, optional Subset of columns to select, denoted either by column labels or column indices. If list-like, all elements must either be positional (i.e. integer indices into the document columns) or strings that correspond to column names provided either by the user in ``names`` or inferred from the document header row(s). If ``names`` are given, the document header row(s) are not taken into account. For example, a valid list-like ``usecols`` parameter would be ``[0, 1, 2]`` or ``['foo', 'bar', 'baz']``. Element order is ignored, so ``usecols=[0, 1]`` is the same as ``[1, 0]``. To instantiate a :class:`~pandas.DataFrame` from ``data`` with element order preserved use ``pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]`` for columns in ``['foo', 'bar']`` order or ``pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]`` for ``['bar', 'foo']`` order. If callable, the callable function will be evaluated against the column names, returning names where the callable function evaluates to ``True``. An example of a valid callable argument would be ``lambda x: x.upper() in ['AAA', 'BBB', 'DDD']``. Using this parameter results in much faster parsing time and lower memory usage. dtype : dtype or dict of {Hashable : dtype}, optional Data type(s) to apply to either the whole dataset or individual columns. E.g., ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}`` Use ``str`` or ``object`` together with suitable ``na_values`` settings to preserve and not interpret ``dtype``. Specifying a ``dtype`` does not change how missing values are parsed; values recognized as missing (see ``na_values`` and ``keep_default_na``) are still read as ``NaN`` (or the dtype's NA value) rather than cast to the requested ``dtype``. If ``converters`` are specified, they will be applied INSTEAD of ``dtype`` conversion. Specify a ``defaultdict`` as input where the default determines the ``dtype`` of the columns which are not explicitly listed. engine : {'c', 'python', 'pyarrow'}, optional Parser engine to use. The C and pyarrow engines are faster, while the python engine is currently more feature-complete. The pyarrow engine is multithreaded, and the C engine reads sufficiently large files in parallel. Some features of the "pyarrow" engine are unsupported or may not work correctly. converters : dict of {Hashable : Callable}, optional Functions for converting values in specified columns. Keys can either be column labels or column indices. The function is applied to the raw text read from the file, so an empty field is passed as an empty string ``''``; ``na_values`` and ``keep_default_na`` are then applied to the value the function returns. true_values : list, optional Values to consider as ``True`` in addition to case-insensitive variants of 'True'. false_values : list, optional Values to consider as ``False`` in addition to case-insensitive variants of 'False'. skipinitialspace : bool, default False Skip spaces after delimiter. skiprows : int, list of int or Callable, optional Line numbers to skip (0-indexed) or number of lines to skip (``int``) at the start of the file. If callable, the callable function will be evaluated against the row indices, returning ``True`` if the row should be skipped and ``False`` otherwise. An example of a valid callable argument would be ``lambda x: x in [0, 2]``. skipfooter : int, default 0 Number of lines at bottom of file to skip (Unsupported with ``engine='c'``). nrows : int, optional Number of rows of file to read. Useful for reading pieces of large files. Refers to the number of data rows in the returned DataFrame, excluding: * The header row containing column names. * Rows before the header row, if ``header=1`` or larger. Example usage: * To read the first 999,999 (non-header) rows: ``read_csv(..., nrows=999999)`` * To read rows 1,000,000 through 1,999,999: ``read_csv(..., skiprows=1000000, nrows=999999)`` na_values : Hashable, Iterable of Hashable or dict of {Hashable : Iterable}, optional Additional strings to recognize as ``NA``/``NaN``. If ``dict`` passed, specific per-column ``NA`` values. By default the following values are interpreted as ``NaN``: empty string, "NaN", "N/A", "NULL", and other common representations of missing data. keep_default_na : bool, default True Whether or not to include the default ``NaN`` values when parsing the data. Depending on whether ``na_values`` is passed in, the behavior is as follows: * If ``keep_default_na`` is ``True``, and ``na_values`` are specified, ``na_values`` is appended to the default ``NaN`` values used for parsing. * If ``keep_default_na`` is ``True``, and ``na_values`` are not specified, only the default ``NaN`` values are used for parsing. * If ``keep_default_na`` is ``False``, and ``na_values`` are specified, only the ``NaN`` values specified ``na_values`` are used for parsing. * If ``keep_default_na`` is ``False``, and ``na_values`` are not specified, no strings will be parsed as ``NaN``. Note that if ``na_filter`` is passed in as ``False``, the ``keep_default_na`` and ``na_values`` parameters will be ignored. na_filter : bool, default True Detect missing value markers (empty strings and the value of ``na_values``). In data without any ``NA`` values, passing ``na_filter=False`` can improve the performance of reading a large file. skip_blank_lines : bool, default True If ``True``, skip over blank lines rather than interpreting as ``NaN`` values. parse_dates : bool, None, list of Hashable, default None The behavior is as follows: * ``bool``. If ``True`` -> try parsing the index. * ``None``. Behaves like ``True`` if ``date_format`` is specified. * ``list`` of ``int`` or names. e.g. If ``[1, 2, 3]`` -> try parsing columns 1, 2, 3 each as a separate date column. If a column or index cannot be represented as an array of ``datetime``, say because of an unparsable value or a mixture of timezones, the column or index will be returned unaltered as an ``object`` data type. For non-standard ``datetime`` parsing, use :func:`~pandas.to_datetime` after :func:`~pandas.read_csv`. Note: A fast-path exists for iso8601-formatted dates. date_format : str or dict of column -> format, optional Format to use for parsing dates and/or times when used in conjunction with ``parse_dates``. The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See `strftime documentation `_ for more information on choices, though note that :const:`"%f"`` will parse all the way up to nanoseconds. You can also pass: - "ISO8601", to parse any `ISO8601 `_ time string (not necessarily in exactly the same format); - "mixed", to infer the format for each element individually. This is risky, and you should probably use it along with `dayfirst`. .. versionadded:: 2.0.0 dayfirst : bool, default False DD/MM format dates, international and European format. cache_dates : bool, default True If ``True``, use a cache of unique, converted dates to apply the ``datetime`` conversion. May produce significant speed-up when parsing duplicate date strings, especially ones with timezone offsets. iterator : bool, default False Return ``TextFileReader`` object for iteration or getting chunks with ``get_chunk()``. chunksize : int, optional Number of lines to read from the file per chunk. Passing a value will cause the function to return a ``TextFileReader`` object for iteration. See the `IO Tools docs `_ for more information on ``iterator`` and ``chunksize``. compression : str or dict, default 'infer' For on-the-fly decompression of on-disk data. If 'infer' and 'filepath_or_buffer' is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression). If using 'zip' or 'tar', the ZIP file must contain only one data file to be read in. Set to ``None`` for no decompression. Can also be a dict with key ``'method'`` set to one of {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``} and other key-value pairs are forwarded to ``zipfile.ZipFile``, ``gzip.GzipFile``, ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or ``tarfile.TarFile``, respectively. As an example, the following could be passed for Zstandard decompression using a custom compression dictionary: ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``. thousands : str (length 1), optional Character acting as the thousands separator in numerical values. decimal : str (length 1), default '.' Character to recognize as decimal point (e.g., use ',' for European data). lineterminator : str (length 1), optional Character used to denote a line break. Only valid with C parser. quotechar : str (length 1), optional Character used to denote the start and end of a quoted item. Quoted items can include the ``delimiter`` and it will be ignored. quoting : {0, 1, 2, 3}, default csv.QUOTE_MINIMAL Control field quoting behavior per ``csv.QUOTE_*`` constants. Use one of ``0`` or ``csv.QUOTE_MINIMAL``, ``1`` or ``csv.QUOTE_ALL``, ``2`` or ``csv.QUOTE_NONNUMERIC``, or ``3`` or ``csv.QUOTE_NONE``. Default is ``csv.QUOTE_MINIMAL`` (i.e., 0) which implies that only fields containing special characters are quoted (e.g., characters defined in ``quotechar``, ``delimiter``, or ``lineterminator``. doublequote : bool, default True When ``quotechar`` is specified and ``quoting`` is not ``QUOTE_NONE``, indicate whether or not to interpret two consecutive ``quotechar`` elements INSIDE a field as a single ``quotechar`` element. escapechar : str (length 1), optional Character used to escape other characters. comment : str (length 1), optional Character indicating that the remainder of line should not be parsed. If found at the beginning of a line, the line will be ignored altogether. This parameter must be a single character. Like empty lines (as long as ``skip_blank_lines=True``), fully commented lines are ignored by the parameter ``header`` but not by ``skiprows``. For example, if ``comment='#'``, parsing ``#empty\\na,b,c\\n1,2,3`` with ``header=0`` will result in ``'a,b,c'`` being treated as the header. encoding : str, optional, default 'utf-8' Encoding to use for UTF when reading/writing (ex. ``'utf-8'``). `List of Python standard encodings `_ . encoding_errors : str, optional, default 'strict' How encoding errors are treated. `List of possible values `_ . dialect : str or csv.Dialect, optional If provided, this parameter will override values (default or not) for the following parameters: ``delimiter``, ``doublequote``, ``escapechar``, ``skipinitialspace``, ``quotechar``, and ``quoting``. If it is necessary to override values, a ``ParserWarning`` will be issued. See ``csv.Dialect`` documentation for more details. on_bad_lines : {'error', 'warn', 'skip'} or Callable, default 'error' Specifies what to do upon encountering a bad line (a line with too many fields). Allowed values are: - ``'error'``, raise an Exception when a bad line is encountered. - ``'warn'``, raise a warning when a bad line is encountered and skip that line. - ``'skip'``, skip bad lines without raising or warning when they are encountered. - Callable, function that will process a single bad line. - With ``engine='python'``, function with signature ``(bad_line: list[str]) -> list[str] | None``. ``bad_line`` is a list of strings split by the ``sep``. If the function returns ``None``, the bad line will be ignored. If the function returns a new ``list`` of strings with more elements than expected, a ``ParserWarning`` will be emitted while dropping extra elements. - With ``engine='pyarrow'``, function with signature as described in pyarrow documentation: `invalid_row_handler `_. .. versionchanged:: 2.2.0 Callable for ``engine='pyarrow'`` low_memory : bool, default True Internally process the file in chunks, resulting in lower memory use while parsing, but possibly mixed type inference. Because each chunk is type-inferred independently, the same literal can be parsed as an ``int`` in one chunk and a ``str`` in another; after concatenation the column has ``object`` dtype and contains both representations, so comparisons such as ``df["col"] == 12345`` or ``df["col"] == "12345"`` will match only a subset of the rows holding that value. A :class:`~pandas.errors.DtypeWarning` is emitted when this occurs. To ensure no mixed types either set ``False``, or specify the type with the ``dtype`` parameter. Note that the entire file is read into a single :class:`~pandas.DataFrame` regardless, use the ``chunksize`` or ``iterator`` parameter to return the data in chunks. (Only valid with C parser). memory_map : bool, default False If a filepath is provided for ``filepath_or_buffer``, map the file object directly onto memory and access the data directly from there. Using this option can improve performance because there is no longer any I/O overhead. float_precision : {'high', 'legacy', 'round_trip'}, optional Specifies which converter the C engine should use for floating-point values. The options are ``None`` or ``'high'`` for the ordinary converter, ``'legacy'`` for the original lower precision pandas converter, and ``'round_trip'`` for the round-trip converter. .. deprecated:: 3.1.0 All float precision modes now use the same converter. The ``float_precision`` argument will be removed in a future version. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib.request.Request`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details, and for more examples on storage options refer `here `_. dtype_backend : {'numpy_nullable', 'pyarrow'} Back-end data type applied to the resultant :class:`DataFrame` (still experimental). If not specified, the default behavior is to not use nullable data types. If specified, the behavior is as follows: * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame` * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype` :class:`DataFrame` .. versionadded:: 2.0 Returns ------- DataFrame or TextFileReader A comma-separated values (csv) file is returned as two-dimensional data structure with labeled axes. See Also -------- DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file. read_table : Read general delimited file into DataFrame. read_fwf : Read a table of fixed-width formatted lines into DataFrame. Notes ----- Column labels read from a header row are always strings. To use column labels of another type, pass them explicitly with ``names`` and set ``header=0`` when the file contains a header row. Sufficiently large local uncompressed files read with the C engine may be parsed by multiple threads in parallel. Use the ``mode.max_threads`` option to cap or disable this; see :ref:`io.csv.parallel` for details. Examples -------- >>> pd.read_csv("data.csv") # doctest: +SKIP Name Value 0 foo 1 1 bar 2 2 #baz 3 Index and header can be specified via the `index_col` and `header` arguments. >>> pd.read_csv("data.csv", header=None) # doctest: +SKIP 0 1 0 Name Value 1 foo 1 2 bar 2 3 #baz 3 >>> pd.read_csv("data.csv", index_col="Value") # doctest: +SKIP Name Value 1 foo 2 bar 3 #baz Column types are inferred but can be explicitly specified using the dtype argument. >>> pd.read_csv("data.csv", dtype={"Value": float}) # doctest: +SKIP Name Value 0 foo 1.0 1 bar 2.0 2 #baz 3.0 True, False, and NA values, and thousands separators have defaults, but can be explicitly specified, too. Supply the values you would like as strings or lists of strings! >>> pd.read_csv("data.csv", na_values=["foo", "bar"]) # doctest: +SKIP Name Value 0 NaN 1 1 NaN 2 2 #baz 3 Comment lines in the input file can be skipped using the `comment` argument. >>> pd.read_csv("data.csv", comment="#") # doctest: +SKIP Name Value 0 foo 1 1 bar 2 By default, columns with dates will be read as ``object`` rather than ``datetime``. >>> df = pd.read_csv("tmp.csv") # doctest: +SKIP >>> df # doctest: +SKIP col 1 col 2 col 3 0 10 10/04/2018 Sun 15 Jan 2023 1 20 15/04/2018 Fri 12 May 2023 >>> df.dtypes # doctest: +SKIP col 1 int64 col 2 object col 3 object dtype: object Specific columns can be parsed as dates by using the `parse_dates` and `date_format` arguments. >>> df = pd.read_csv( ... "tmp.csv", ... parse_dates=[1, 2], ... date_format={"col 2": "%d/%m/%Y", "col 3": "%a %d %b %Y"}, ... ) # doctest: +SKIP >>> df.dtypes # doctest: +SKIP col 1 int64 col 2 datetime64[ns] col 3 datetime64[ns] dtype: object """ # locals() should never be modified kwds = locals().copy() del kwds["filepath_or_buffer"] del kwds["sep"] kwds_defaults = _refine_defaults_read( dialect, delimiter, engine, sep, on_bad_lines, names, defaults={"delimiter": ","}, dtype_backend=dtype_backend, lineterminator=lineterminator, ) kwds.update(kwds_defaults) return _read(filepath_or_buffer, kwds) @overload def read_table( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: Literal[True], chunksize: int | None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> TextFileReader: ... @overload def read_table( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: bool = ..., chunksize: int, **kwds: Unpack[_read_shared[HashableT]], ) -> TextFileReader: ... @overload def read_table( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: Literal[False] = ..., chunksize: None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> DataFrame: ... @overload def read_table( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, iterator: bool = ..., chunksize: int | None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> DataFrame | TextFileReader: ... @set_module("pandas") def read_table( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, sep: str | lib.NoDefault | None = lib.no_default, delimiter: str | lib.NoDefault | None = lib.no_default, # Column and Index Locations and Names header: int | Sequence[int] | Literal["infer"] | None = "infer", names: Sequence[Hashable] | lib.NoDefault | None = lib.no_default, index_col: IndexLabel | Literal[False] | None = None, usecols: UsecolsArgType = None, # General Parsing Configuration dtype: DtypeArg | None = None, engine: CSVEngine | None = None, converters: Mapping[HashableT, Callable] | None = None, true_values: list | None = None, false_values: list | None = None, skipinitialspace: bool = False, skiprows: list[int] | int | Callable[[Hashable], bool] | None = None, skipfooter: int = 0, nrows: int | None = None, # NA and Missing Data Handling na_values: ( Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None ) = None, keep_default_na: bool = True, na_filter: bool = True, skip_blank_lines: bool = True, # Datetime Handling parse_dates: bool | Sequence[Hashable] | None = None, date_format: str | dict[Hashable, str] | None = None, dayfirst: bool = False, cache_dates: bool = True, # Iteration iterator: bool = False, chunksize: int | None = None, # Quoting, Compression, and File Format compression: CompressionOptions = "infer", thousands: str | None = None, decimal: str = ".", lineterminator: str | None = None, quotechar: str = '"', quoting: int = csv.QUOTE_MINIMAL, doublequote: bool = True, escapechar: str | None = None, comment: str | None = None, encoding: str | None = None, encoding_errors: str | None = "strict", dialect: str | csv.Dialect | type[csv.Dialect] | None = None, # Error Handling on_bad_lines: str = "error", # Internal low_memory: bool = _c_parser_defaults["low_memory"], memory_map: bool = False, float_precision: Literal["high", "legacy", "round_trip"] | None = None, storage_options: StorageOptions | None = None, dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default, ) -> DataFrame | TextFileReader: """ Read general delimited file into DataFrame. Also supports optionally iterating or breaking of the file into chunks. Additional help can be found in the online docs for `IO Tools `_. Parameters ---------- filepath_or_buffer : str, path object or file-like object Any valid string path is acceptable. The string could be a URL. Valid URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is expected. A local file could be: file://localhost/path/to/table.csv. Certain URL schemes may require additional packages. For example, S3 URLs require the ``s3fs`` library. See :ref:`install.optional_dependencies` for a full list. If you want to pass in a path object, pandas accepts any ``os.PathLike``. By file-like object, we refer to objects with a ``read()`` method that accepts an optional size argument, such as a file handle (e.g. via builtin ``open`` function) or ``StringIO``. sep : str, default '\\t' (tab-stop) Character or regex pattern to treat as the delimiter. ``sep=None`` detects the separator from the first valid row of the file with Python's builtin sniffer tool, ``csv.Sniffer``; it is supported only by the Python parsing engine, which will be used automatically. In addition, separators longer than 1 character and different from ``'\\s+'`` will be interpreted as regular expressions and will force the use of the Python parsing engine. Note that regex delimiters are prone to ignoring quoted data. Regex example: ``'\\r\\t'``. delimiter : str, optional Alias for ``sep``. header : int, Sequence of int, 'infer' or None, default 'infer' Row index or indices in the file to use as DataFrame column labels. Indexing starts at 0 and counts only non-blank, non-commented lines when ``skip_blank_lines=True``, so ``header=0`` denotes the first line of data rather than the first line of the file. Valid arguments are: * ``int``: line index at which column labels are read. * Sequence of ``int``: line indices at which column labels are read and combined into a :class:`~pandas.MultiIndex` on the columns, e.g. ``[0, 1, 3]``. Intervening rows that are not specified will be skipped (e.g. row 2 in this example). * ``None``: no row in the file is interpreted as column labels. Columns are labelled by integer position, or by the values passed to ``names`` when provided. Use this for files with no header row. If the file has a header row that should be overridden by ``names``, pass ``header=0`` instead. * ``'infer'`` (default): equivalent to ``header=0`` when no ``names`` are passed, otherwise equivalent to ``header=None``. When inferred from the file contents, headers are kept distinct from each other by renaming duplicate names with a numeric suffix of the form ``".{count}"`` starting from 1, e.g. ``"foo"`` and ``"foo.1"``. Empty headers are named ``"Unnamed: {i}"`` or ``"Unnamed: {i}_level_{level}"`` in the case of MultiIndex columns. names : Sequence of Hashable, optional Sequence of column labels to apply. If the file contains a header row, then you should explicitly pass ``header=0`` to override the column names. Duplicates in this list are not allowed. index_col : Hashable, Sequence of Hashable or False, optional Column(s) to use as row label(s), denoted either by column labels or column indices. If a sequence of labels or indices is given, :class:`~pandas.MultiIndex` will be formed for the row labels. Note: ``index_col=False`` can be used to force pandas to *not* use the first column as the index, e.g., when you have a malformed file with delimiters at the end of each line. usecols : Sequence of Hashable or Callable, optional Subset of columns to select, denoted either by column labels or column indices. If list-like, all elements must either be positional (i.e. integer indices into the document columns) or strings that correspond to column names provided either by the user in ``names`` or inferred from the document header row(s). If ``names`` are given, the document header row(s) are not taken into account. For example, a valid list-like ``usecols`` parameter would be ``[0, 1, 2]`` or ``['foo', 'bar', 'baz']``. Element order is ignored, so ``usecols=[0, 1]`` is the same as ``[1, 0]``. To instantiate a :class:`~pandas.DataFrame` from ``data`` with element order preserved use ``pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]`` for columns in ``['foo', 'bar']`` order or ``pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]`` for ``['bar', 'foo']`` order. If callable, the callable function will be evaluated against the column names, returning names where the callable function evaluates to ``True``. An example of a valid callable argument would be ``lambda x: x.upper() in ['AAA', 'BBB', 'DDD']``. Using this parameter results in much faster parsing time and lower memory usage. dtype : dtype or dict of {Hashable : dtype}, optional Data type(s) to apply to either the whole dataset or individual columns. E.g., ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}`` Use ``str`` or ``object`` together with suitable ``na_values`` settings to preserve and not interpret ``dtype``. Specifying a ``dtype`` does not change how missing values are parsed; values recognized as missing (see ``na_values`` and ``keep_default_na``) are still read as ``NaN`` (or the dtype's NA value) rather than cast to the requested ``dtype``. If ``converters`` are specified, they will be applied INSTEAD of ``dtype`` conversion. Specify a ``defaultdict`` as input where the default determines the ``dtype`` of the columns which are not explicitly listed. engine : {'c', 'python', 'pyarrow'}, optional Parser engine to use. The C and pyarrow engines are faster, while the python engine is currently more feature-complete. The pyarrow engine is multithreaded, and the C engine reads sufficiently large files in parallel. The 'pyarrow' engine is an *experimental* engine, and some features are unsupported, or may not work correctly, with this engine. converters : dict of {Hashable : Callable}, optional Functions for converting values in specified columns. Keys can either be column labels or column indices. The function is applied to the raw text read from the file, so an empty field is passed as an empty string ``''``; ``na_values`` and ``keep_default_na`` are then applied to the value the function returns. true_values : list, optional Values to consider as ``True`` in addition to case-insensitive variants of 'True'. false_values : list, optional Values to consider as ``False`` in addition to case-insensitive variants of 'False'. skipinitialspace : bool, default False Skip spaces after delimiter. skiprows : int, list of int or Callable, optional Line numbers to skip (0-indexed) or number of lines to skip (``int``) at the start of the file. If callable, the callable function will be evaluated against the row indices, returning ``True`` if the row should be skipped and ``False`` otherwise. An example of a valid callable argument would be ``lambda x: x in [0, 2]``. skipfooter : int, default 0 Number of lines at bottom of file to skip (Unsupported with ``engine='c'``). nrows : int, optional Number of rows of file to read. Useful for reading pieces of large files. Refers to the number of data rows in the returned DataFrame, excluding: * The header row containing column names. * Rows before the header row, if ``header=1`` or larger. Example usage: * To read the first 999,999 (non-header) rows: ``read_csv(..., nrows=999999)`` * To read rows 1,000,000 through 1,999,999: ``read_csv(..., skiprows=1000000, nrows=999999)`` na_values : Hashable, Iterable of Hashable or dict of {Hashable : Iterable}, optional Additional strings to recognize as ``NA``/``NaN``. If ``dict`` passed, specific per-column ``NA`` values. By default the following values are interpreted as ``NaN``: empty string, "NaN", "N/A", "NULL", and other common representations of missing data. keep_default_na : bool, default True Whether or not to include the default ``NaN`` values when parsing the data. Depending on whether ``na_values`` is passed in, the behavior is as follows: * If ``keep_default_na`` is ``True``, and ``na_values`` are specified, ``na_values`` is appended to the default ``NaN`` values used for parsing. * If ``keep_default_na`` is ``True``, and ``na_values`` are not specified, only the default ``NaN`` values are used for parsing. * If ``keep_default_na`` is ``False``, and ``na_values`` are specified, only the ``NaN`` values specified ``na_values`` are used for parsing. * If ``keep_default_na`` is ``False``, and ``na_values`` are not specified, no strings will be parsed as ``NaN``. Note that if ``na_filter`` is passed in as ``False``, the ``keep_default_na`` and ``na_values`` parameters will be ignored. na_filter : bool, default True Detect missing value markers (empty strings and the value of ``na_values``). In data without any ``NA`` values, passing ``na_filter=False`` can improve the performance of reading a large file. skip_blank_lines : bool, default True If ``True``, skip over blank lines rather than interpreting as ``NaN`` values. parse_dates : bool, None, list of Hashable, default None The behavior is as follows: * ``bool``. If ``True`` -> try parsing the index. * ``None``. Behaves like ``True`` if ``date_format`` is specified. * ``list`` of ``int`` or names. e.g. If ``[1, 2, 3]`` -> try parsing columns 1, 2, 3 each as a separate date column. If a column or index cannot be represented as an array of ``datetime``, say because of an unparsable value or a mixture of timezones, the column or index will be returned unaltered as an ``object`` data type. For non-standard ``datetime`` parsing, use :func:`~pandas.to_datetime` after :func:`~pandas.read_csv`. Note: A fast-path exists for iso8601-formatted dates. date_format : str or dict of column -> format, optional Format to use for parsing dates and/or times when used in conjunction with ``parse_dates``. The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See `strftime documentation `_ for more information on choices, though note that :const:`"%f"`` will parse all the way up to nanoseconds. You can also pass: - "ISO8601", to parse any `ISO8601 `_ time string (not necessarily in exactly the same format); - "mixed", to infer the format for each element individually. This is risky, and you should probably use it along with `dayfirst`. .. versionadded:: 2.0.0 dayfirst : bool, default False DD/MM format dates, international and European format. cache_dates : bool, default True If ``True``, use a cache of unique, converted dates to apply the ``datetime`` conversion. May produce significant speed-up when parsing duplicate date strings, especially ones with timezone offsets. iterator : bool, default False Return ``TextFileReader`` object for iteration or getting chunks with ``get_chunk()``. chunksize : int, optional Number of lines to read from the file per chunk. Passing a value will cause the function to return a ``TextFileReader`` object for iteration. See the `IO Tools docs `_ for more information on ``iterator`` and ``chunksize``. compression : str or dict, default 'infer' For on-the-fly decompression of on-disk data. If 'infer' and 'filepath_or_buffer' is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression). If using 'zip' or 'tar', the ZIP file must contain only one data file to be read in. Set to ``None`` for no decompression. Can also be a dict with key ``'method'`` set to one of {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``} and other key-value pairs are forwarded to ``zipfile.ZipFile``, ``gzip.GzipFile``, ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or ``tarfile.TarFile``, respectively. As an example, the following could be passed for Zstandard decompression using a custom compression dictionary: ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``. thousands : str (length 1), optional Character acting as the thousands separator in numerical values. decimal : str (length 1), default '.' Character to recognize as decimal point (e.g., use ',' for European data). lineterminator : str (length 1), optional Character used to denote a line break. Only valid with C parser. quotechar : str (length 1), optional Character used to denote the start and end of a quoted item. Quoted items can include the ``delimiter`` and it will be ignored. quoting : {0, 1, 2, 3}, default csv.QUOTE_MINIMAL Control field quoting behavior per ``csv.QUOTE_*`` constants. Use one of ``0`` or ``csv.QUOTE_MINIMAL``, ``1`` or ``csv.QUOTE_ALL``, ``2`` or ``csv.QUOTE_NONNUMERIC``, or ``3`` or ``csv.QUOTE_NONE``. Default is ``csv.QUOTE_MINIMAL`` (i.e., 0) which implies that only fields containing special characters are quoted (e.g., characters defined in ``quotechar``, ``delimiter``, or ``lineterminator``. doublequote : bool, default True When ``quotechar`` is specified and ``quoting`` is not ``QUOTE_NONE``, indicate whether or not to interpret two consecutive ``quotechar`` elements INSIDE a field as a single ``quotechar`` element. escapechar : str (length 1), optional Character used to escape other characters. comment : str (length 1), optional Character indicating that the remainder of line should not be parsed. If found at the beginning of a line, the line will be ignored altogether. This parameter must be a single character. Like empty lines (as long as ``skip_blank_lines=True``), fully commented lines are ignored by the parameter ``header`` but not by ``skiprows``. For example, if ``comment='#'``, parsing ``#empty\\na,b,c\\n1,2,3`` with ``header=0`` will result in ``'a,b,c'`` being treated as the header. encoding : str, optional, default 'utf-8' Encoding to use for UTF when reading/writing (ex. ``'utf-8'``). `List of Python standard encodings `_ . encoding_errors : str, optional, default 'strict' How encoding errors are treated. `List of possible values `_ . dialect : str or csv.Dialect, optional If provided, this parameter will override values (default or not) for the following parameters: ``delimiter``, ``doublequote``, ``escapechar``, ``skipinitialspace``, ``quotechar``, and ``quoting``. If it is necessary to override values, a ``ParserWarning`` will be issued. See ``csv.Dialect`` documentation for more details. on_bad_lines : {'error', 'warn', 'skip'} or Callable, default 'error' Specifies what to do upon encountering a bad line (a line with too many fields). Allowed values are: - ``'error'``, raise an Exception when a bad line is encountered. - ``'warn'``, raise a warning when a bad line is encountered and skip that line. - ``'skip'``, skip bad lines without raising or warning when they are encountered. - Callable, function that will process a single bad line. - With ``engine='python'``, function with signature ``(bad_line: list[str]) -> list[str] | None``. ``bad_line`` is a list of strings split by the ``sep``. If the function returns ``None``, the bad line will be ignored. If the function returns a new ``list`` of strings with more elements than expected, a ``ParserWarning`` will be emitted while dropping extra elements. - With ``engine='pyarrow'``, function with signature as described in pyarrow documentation: `invalid_row_handler `_. .. versionadded:: 2.2.0 Callable for ``engine='pyarrow'`` low_memory : bool, default True Internally process the file in chunks, resulting in lower memory use while parsing, but possibly mixed type inference. Because each chunk is type-inferred independently, the same literal can be parsed as an ``int`` in one chunk and a ``str`` in another; after concatenation the column has ``object`` dtype and contains both representations, so comparisons such as ``df["col"] == 12345`` or ``df["col"] == "12345"`` will match only a subset of the rows holding that value. A :class:`~pandas.errors.DtypeWarning` is emitted when this occurs. To ensure no mixed types either set ``False``, or specify the type with the ``dtype`` parameter. Note that the entire file is read into a single :class:`~pandas.DataFrame` regardless, use the ``chunksize`` or ``iterator`` parameter to return the data in chunks. (Only valid with C parser). memory_map : bool, default False If a filepath is provided for ``filepath_or_buffer``, map the file object directly onto memory and access the data directly from there. Using this option can improve performance because there is no longer any I/O overhead. float_precision : {'high', 'legacy', 'round_trip'}, optional Specifies which converter the C engine should use for floating-point values. The options are ``None`` or ``'high'`` for the ordinary converter, ``'legacy'`` for the original lower precision pandas converter, and ``'round_trip'`` for the round-trip converter. .. deprecated:: 3.1.0 All float precision modes now use the same converter. The ``float_precision`` argument will be removed in a future version. storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc. For HTTP(S) URLs the key-value pairs are forwarded to ``urllib.request.Request`` as header options. For other URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more details, and for more examples on storage options refer `here `_. dtype_backend : {'numpy_nullable', 'pyarrow'} Back-end data type applied to the resultant :class:`DataFrame` (still experimental). If not specified, the default behavior is to not use nullable data types. If specified, the behavior is as follows: * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame` * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype` :class:`DataFrame` .. versionadded:: 2.0 Returns ------- DataFrame or TextFileReader A comma-separated values (csv) file is returned as two-dimensional data structure with labeled axes. See Also -------- DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file. read_csv : Read a comma-separated values (csv) file into DataFrame. read_fwf : Read a table of fixed-width formatted lines into DataFrame. Notes ----- Column labels read from a header row are always strings. To use column labels of another type, pass them explicitly with ``names`` and set ``header=0`` when the file contains a header row. Examples -------- >>> pd.read_table("data.csv") # doctest: +SKIP Name Value 0 foo 1 1 bar 2 2 #baz 3 Index and header can be specified via the `index_col` and `header` arguments. >>> pd.read_table("data.csv", header=None) # doctest: +SKIP 0 1 0 Name Value 1 foo 1 2 bar 2 3 #baz 3 >>> pd.read_table("data.csv", index_col="Value") # doctest: +SKIP Name Value 1 foo 2 bar 3 #baz Column types are inferred but can be explicitly specified using the dtype argument. >>> pd.read_table("data.csv", dtype={"Value": float}) # doctest: +SKIP Name Value 0 foo 1.0 1 bar 2.0 2 #baz 3.0 True, False, and NA values, and thousands separators have defaults, but can be explicitly specified, too. Supply the values you would like as strings or lists of strings! >>> pd.read_table("data.csv", na_values=["foo", "bar"]) # doctest: +SKIP Name Value 0 NaN 1 1 NaN 2 2 #baz 3 Comment lines in the input file can be skipped using the `comment` argument. >>> pd.read_table("data.csv", comment="#") # doctest: +SKIP Name Value 0 foo 1 1 bar 2 By default, columns with dates will be read as ``object`` rather than ``datetime``. >>> df = pd.read_table("tmp.csv") # doctest: +SKIP >>> df # doctest: +SKIP col 1 col 2 col 3 0 10 10/04/2018 Sun 15 Jan 2023 1 20 15/04/2018 Fri 12 May 2023 >>> df.dtypes # doctest: +SKIP col 1 int64 col 2 object col 3 object dtype: object Specific columns can be parsed as dates by using the `parse_dates` and `date_format` arguments. >>> df = pd.read_table( ... "tmp.csv", ... parse_dates=[1, 2], ... date_format={"col 2": "%d/%m/%Y", "col 3": "%a %d %b %Y"}, ... ) # doctest: +SKIP >>> df.dtypes # doctest: +SKIP col 1 int64 col 2 datetime64[ns] col 3 datetime64[ns] dtype: object """ # locals() should never be modified kwds = locals().copy() del kwds["filepath_or_buffer"] del kwds["sep"] kwds_defaults = _refine_defaults_read( dialect, delimiter, engine, sep, on_bad_lines, names, defaults={"delimiter": "\t"}, dtype_backend=dtype_backend, lineterminator=lineterminator, ) kwds.update(kwds_defaults) return _read(filepath_or_buffer, kwds) @overload def read_fwf( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, colspecs: Sequence[tuple[int, int]] | str | None = ..., widths: Sequence[int] | None = ..., infer_nrows: int = ..., iterator: Literal[True], chunksize: int | None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> TextFileReader: ... @overload def read_fwf( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, colspecs: Sequence[tuple[int, int]] | str | None = ..., widths: Sequence[int] | None = ..., infer_nrows: int = ..., iterator: bool = ..., chunksize: int, **kwds: Unpack[_read_shared[HashableT]], ) -> TextFileReader: ... @overload def read_fwf( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, colspecs: Sequence[tuple[int, int]] | str | None = ..., widths: Sequence[int] | None = ..., infer_nrows: int = ..., iterator: Literal[False] = ..., chunksize: None = ..., **kwds: Unpack[_read_shared[HashableT]], ) -> DataFrame: ... @set_module("pandas") def read_fwf( filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], *, colspecs: Sequence[tuple[int, int]] | str | None = "infer", widths: Sequence[int] | None = None, infer_nrows: int = 100, iterator: bool = False, chunksize: int | None = None, **kwds: Unpack[_read_shared[HashableT]], ) -> DataFrame | TextFileReader: r""" Read a table of fixed-width formatted lines into DataFrame. Also supports optionally iterating or breaking of the file into chunks. Additional help can be found in the `online docs for IO Tools `_. Parameters ---------- filepath_or_buffer : str, path object, or file-like object Any valid string path is acceptable. The string could be a URL. Valid URL schemes include http, ftp, s3, and file. For file URLs, a host is expected. A local file could be: ``file://localhost/path/to/table.csv``. Certain URL schemes may require additional packages. For example, S3 URLs require the ``s3fs`` library. See :ref:`install.optional_dependencies` for a full list. If you want to pass in a path object, pandas accepts any ``os.PathLike``. By file-like object, we refer to objects with a ``read()`` method that accepts an optional size argument, such as a file handle (e.g. via builtin ``open`` function) or ``StringIO``. colspecs : list of tuple (int, int) or 'infer'. optional A list of tuples giving the extents of the fixed-width fields of each line as half-open intervals (i.e., [from, to] ). String value 'infer' can be used to instruct the parser to try detecting the column specifications from the first 100 rows of the data which are not being skipped via skiprows (default='infer'). widths : list of int, optional A list of field widths which can be used instead of 'colspecs' if the intervals are contiguous. infer_nrows : int, default 100 The number of rows to consider when letting the parser determine the ``colspecs``. Pass ``np.inf`` to infer column specifications from the entire file, which is useful when columns vary in width and a value wider than anything in the first ``infer_nrows`` rows would otherwise be truncated. iterator : bool, default False Return ``TextFileReader`` object for iteration or getting chunks with ``get_chunk()``. chunksize : int, optional Number of lines to read from the file per chunk. **kwds : optional Optional keyword arguments can be passed to ``TextFileReader``. Returns ------- DataFrame or TextFileReader A comma-separated values (csv) file is returned as two-dimensional data structure with labeled axes. See Also -------- DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file. read_csv : Read a comma-separated values (csv) file into DataFrame. Examples -------- >>> pd.read_fwf("data.csv") # doctest: +SKIP """ # Check input arguments. if colspecs is None and widths is None: raise ValueError("Must specify either colspecs or widths") if colspecs not in (None, "infer") and widths is not None: raise ValueError("You must specify only one of 'widths' and 'colspecs'") # Compute 'colspecs' from 'widths', if specified. if widths is not None: colspecs, col = [], 0 for w in widths: colspecs.append((col, col + w)) col += w # for mypy assert colspecs is not None # GH#40830 # Ensure length of `colspecs` matches length of `names` names = kwds.get("names") if names is not None and names is not lib.no_default: if len(names) != len(colspecs) and colspecs != "infer": # need to check len(index_col) as it might contain # unnamed indices, in which case it's name is not required len_index = 0 if kwds.get("index_col") is not None: index_col: Any = kwds.get("index_col") if index_col is not False: if not is_list_like(index_col): len_index = 1 else: # for mypy: handled in the if-branch assert index_col is not lib.no_default len_index = len(index_col) if kwds.get("usecols") is None and len(names) + len_index != len(colspecs): # If usecols is used colspec may be longer than names raise ValueError("Length of colspecs must match length of names") check_dtype_backend(kwds.setdefault("dtype_backend", lib.no_default)) return _read( filepath_or_buffer, kwds | { "colspecs": colspecs, "infer_nrows": infer_nrows, "engine": "python-fwf", "iterator": iterator, "chunksize": chunksize, }, ) class TextFileReader(abc.Iterator): """ Iterator over chunks of a delimited text file. Returned by :func:`read_csv`, :func:`read_table`, and :func:`read_fwf` when ``iterator=True`` or ``chunksize`` is given. Not intended to be constructed directly by users. A ``TextFileReader`` can be iterated over to yield successive chunks as :class:`DataFrame` objects, used as a context manager to ensure the underlying file handle is closed, or queried explicitly via :meth:`~TextFileReader.read` and :meth:`~TextFileReader.get_chunk`. Passed ``dialect`` overrides any of the related parser options. Parameters ---------- f : str, path object, or file-like object Source to read from. Accepts the same inputs as :func:`read_csv`. engine : {'c', 'python', 'pyarrow', 'python-fwf'}, optional Parser engine to use. If not specified, defaults to ``'python'``. **kwds Any keyword argument accepted by :func:`read_csv`, :func:`read_table`, or :func:`read_fwf`. See Also -------- read_csv : Read a comma-separated values (csv) file into a DataFrame. read_table : Read general delimited file into a DataFrame. read_fwf : Read a table of fixed-width formatted lines into a DataFrame. Examples -------- >>> with pd.read_csv("data.csv", chunksize=1000) as reader: # doctest: +SKIP ... for chunk in reader: ... process(chunk) >>> with pd.read_csv("data.csv", iterator=True) as reader: # doctest: +SKIP ... first = reader.get_chunk(5) """ def __init__( self, f: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str] | list, engine: CSVEngine | None = None, **kwds, ) -> None: if engine is not None: engine_specified = True else: engine = "python" engine_specified = False self.engine = engine self._engine_specified = kwds.get("engine_specified", engine_specified) # before _validate_skipfooter, which reads `iterator` for truthiness _validate_bool_kwargs(kwds) _validate_skipfooter(kwds) dialect = _extract_dialect(kwds) if dialect is not None: if engine == "pyarrow": raise ValueError( "The 'dialect' option is not supported with the 'pyarrow' engine" ) kwds = _merge_with_dialect_properties(dialect, kwds) # again, for the values the dialect supplied _validate_bool_kwargs(kwds) if kwds.get("header", "infer") == "infer": kwds["header"] = 0 if kwds.get("names") is None else None self.orig_options = kwds # miscellanea self._currow = 0 options = self._get_options_with_defaults(engine) options["storage_options"] = kwds.get("storage_options", None) self.chunksize = options.pop("chunksize", None) self.nrows = options.pop("nrows", None) self._check_file_or_buffer(f, engine) self.options, self.engine = self._clean_options(options, engine) if "has_index_names" in kwds: self.options["has_index_names"] = kwds["has_index_names"] self.handles: IOHandles | None = None self._engine = self._make_engine(f, self.engine) def close(self) -> None: """ Close the underlying file handle and parser engine. Called automatically when the ``TextFileReader`` is used as a context manager. See Also -------- TextFileReader.read : Read rows from the file into a DataFrame. TextFileReader.get_chunk : Read the next chunk of rows. Examples -------- >>> reader = pd.read_csv("data.csv", chunksize=5) # doctest: +SKIP >>> reader.close() # doctest: +SKIP """ if self.handles is not None: self.handles.close() self._engine.close() def _get_options_with_defaults(self, engine: CSVEngine) -> dict[str, Any]: kwds = self.orig_options options = {} default: object | None for argname, default in parser_defaults.items(): value = kwds.get(argname, default) # see gh-12935 if ( engine == "pyarrow" and argname in _pyarrow_unsupported and value != default and value != getattr(value, "value", default) ): raise ValueError( f"The {argname!r} option is not supported with the 'pyarrow' engine" ) options[argname] = value for argname, default in _c_parser_defaults.items(): if argname in kwds: value = kwds[argname] if engine != "c" and value != default: # TODO: Refactor this logic, its pretty convoluted if "python" in engine and argname not in _python_unsupported: pass elif "pyarrow" in engine and argname not in _pyarrow_unsupported: pass else: raise ValueError( f"The {argname!r} option is not supported with the " f"{engine!r} engine" ) else: value = default options[argname] = value if engine == "python-fwf": for argname, default in _fwf_defaults.items(): options[argname] = kwds.get(argname, default) return options def _check_file_or_buffer(self, f, engine: CSVEngine) -> None: if hasattr(f, "encoding"): file_encoding = f.encoding orig_reader_enc = self.orig_options.get("encoding", None) any_none = file_encoding is None or orig_reader_enc is None if file_encoding != orig_reader_enc and not any_none: file_path = getattr(f, "name", None) raise ValueError( f"The specified reader encoding {orig_reader_enc} is different " f"from the encoding {file_encoding} of file {file_path}." ) def _clean_options( self, options: dict[str, Any], engine: CSVEngine ) -> tuple[dict[str, Any], CSVEngine]: result = options.copy() fallback_reason = None # C engine not supported yet if engine == "c": if options["skipfooter"] > 0: fallback_reason = "the 'c' engine does not support skipfooter" engine = "python" sep = options["delimiter"] if sep is None: # sniffing the separator with csv.Sniffer is python-engine only if engine in ("c", "pyarrow"): fallback_reason = f"the '{engine}' engine does not support sep=None" engine = "python" elif len(sep) > 1: if engine == "c" and sep == r"\s+": # delim_whitespace passed on to pandas._libs.parsers.TextReader result["delim_whitespace"] = True del result["delimiter"] elif engine not in ("python", "python-fwf"): # wait until regex engine integrated fallback_reason = ( f"the '{engine}' engine does not support " "separators > 1 char, including regex separators" ) engine = "python" else: # The C engine always tokenizes a utf-8 byte stream: text handles # are re-encoded to utf-8 before reaching it, whatever `encoding` # says. So the separator has to be a single utf-8 byte; the # platform's filesystem encoding has nothing to do with it. encodeable = True try: if len(sep.encode("utf-8")) > 1: encodeable = False except UnicodeEncodeError: # e.g. a lone surrogate encodeable = False if not encodeable and engine not in ("python", "python-fwf"): fallback_reason = ( "the separator encoded in utf-8 " f"is > 1 char long, and the '{engine}' engine " "does not support such separators" ) engine = "python" quotechar = options["quotechar"] if quotechar is not None and isinstance(quotechar, (str, bytes)): if ( len(quotechar) == 1 and ord(quotechar) > 127 and engine not in ("python", "python-fwf") ): fallback_reason = ( "ord(quotechar) > 127, meaning the " "quotechar is larger than one byte, " f"and the '{engine}' engine does not support such quotechars" ) engine = "python" if fallback_reason and self._engine_specified: raise ValueError(fallback_reason) if engine == "c": for arg in _c_unsupported: del result[arg] if "python" in engine: for arg in _python_unsupported: if fallback_reason and result[arg] != _c_parser_defaults.get(arg): raise ValueError( "Falling back to the 'python' engine because " f"{fallback_reason}, but this causes {arg!r} to be " "ignored as it is not supported by the 'python' engine." ) del result[arg] if fallback_reason: warnings.warn( ( "Falling back to the 'python' engine because " f"{fallback_reason}; you can avoid this warning by specifying " "engine='python'." ), ParserWarning, stacklevel=find_stack_level(), ) index_col = options["index_col"] names = options["names"] converters = options["converters"] na_values = options["na_values"] skiprows = options["skiprows"] validate_header_arg(options["header"]) if index_col is True: raise ValueError("The value of index_col couldn't be 'True'") if is_index_col(index_col): if not isinstance(index_col, (list, tuple, np.ndarray)): index_col = [index_col] result["index_col"] = index_col names = list(names) if names is not None else names # type conversion-related if converters is not None: if not isinstance(converters, dict): raise TypeError( "Type converters must be a dict or subclass, " f"input was a {type(converters).__name__}" ) else: converters = {} # Converting values to NA keep_default_na = options["keep_default_na"] floatify = engine != "pyarrow" na_values, na_fvalues = _clean_na_values( na_values, keep_default_na, floatify=floatify ) # handle skiprows; this is internally handled by the # c-engine, so only need for python and pyarrow parsers if engine == "pyarrow": if not is_integer(skiprows) and skiprows is not None: # pyarrow expects skiprows to be passed as an integer raise ValueError( "skiprows argument must be an integer when using engine='pyarrow'" ) else: if is_integer(skiprows): skiprows = range(skiprows) if skiprows is None: skiprows = set() elif not callable(skiprows): skiprows = set(skiprows) # put stuff back result["names"] = names result["converters"] = converters result["na_values"] = na_values result["na_fvalues"] = na_fvalues result["skiprows"] = skiprows return result, engine def __next__(self) -> DataFrame: try: return self.get_chunk() except StopIteration: self.close() raise def _make_engine( self, f: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str] | list | IO, engine: CSVEngine = "c", ) -> ParserBase: mapping: dict[str, type[ParserBase]] = { "c": CParserWrapper, "python": PythonParser, "pyarrow": ArrowParserWrapper, "python-fwf": FixedWidthFieldParser, } if engine not in mapping: raise ValueError( f"Unknown engine: {engine} (valid options are {mapping.keys()})" ) if not isinstance(f, list): # open file here # The c and pyarrow engines can decode utf-8 bytes natively; # wrapping in TextIOWrapper makes them slower, especially for # memory_map and remote file-like objects (GH#46823). is_text = engine in ("python", "python-fwf") or ( engine == "c" and self.options.get("encoding", "utf-8") not in (None, "utf-8") ) mode = "r" if is_text else "rb" self.handles = get_handle( f, mode, encoding=self.options.get("encoding", None), compression=self.options.get("compression", None), memory_map=self.options.get("memory_map", False), is_text=is_text, errors=self.options.get("encoding_errors", "strict"), storage_options=self.options.get("storage_options", None), ) assert self.handles is not None f = self.handles.handle elif engine != "python": msg = f"Invalid file path or buffer object type: {type(f)}" raise ValueError(msg) try: return mapping[engine](f, **self.options) except Exception: if self.handles is not None: self.handles.close() raise def _failover_to_python(self) -> None: raise AbstractMethodError(self) def read(self, nrows: int | None = None) -> DataFrame: """ Read rows from the file into a :class:`DataFrame`. Advances the internal cursor by the number of rows returned, so successive calls yield later rows of the file. Used with readers created via :func:`read_csv` or :func:`read_table` with ``iterator=True`` or ``chunksize`` set. Parameters ---------- nrows : int, optional Number of rows to read. If ``None``, read until end of file. Ignored when the underlying engine is ``"pyarrow"``, which always reads the full file. Returns ------- DataFrame The parsed rows. See Also -------- TextFileReader.get_chunk : Read the next chunk of rows. read_csv : Read a comma-separated values (csv) file into a DataFrame. Examples -------- >>> with pd.read_csv("data.csv", iterator=True) as reader: # doctest: +SKIP ... df = reader.read() """ if self.engine == "pyarrow": try: # error: "ParserBase" has no attribute "read" df = self._engine.read() # type: ignore[attr-defined] except Exception: self.close() raise else: nrows = validate_integer("nrows", nrows) try: # error: "ParserBase" has no attribute "read" ( index, columns, col_dict, ) = self._engine.read( # type: ignore[attr-defined] nrows ) except Exception: self.close() raise if index is None: if col_dict: # Any column is actually fine: new_rows = len(next(iter(col_dict.values()))) index = RangeIndex(self._currow, self._currow + new_rows) else: new_rows = 0 else: new_rows = len(index) if hasattr(self, "orig_options"): dtype_arg = self.orig_options.get("dtype", None) else: dtype_arg = None if isinstance(dtype_arg, dict): dtype = defaultdict(lambda: None) # type: ignore[var-annotated] dtype.update(dtype_arg) elif dtype_arg is not None and pandas_dtype(dtype_arg) in ( np.str_, np.object_, ): dtype = defaultdict(lambda: dtype_arg) else: dtype = None if dtype is not None: new_col_dict = {} for k, v in col_dict.items(): d = ( dtype[k] if pandas_dtype(dtype[k]) in (np.str_, np.object_) else None ) new_col_dict[k] = Series(v, index=index, dtype=d, copy=False) else: new_col_dict = col_dict df = DataFrame( new_col_dict, columns=columns, index=index, copy=False, ) self._currow += new_rows return df def get_chunk(self, size: int | None = None) -> DataFrame: """ Read the next chunk of rows from the file. Convenience wrapper around :meth:`TextFileReader.read` that defaults ``size`` to the ``chunksize`` passed to :func:`read_csv` or :func:`read_table`, and respects a configured ``nrows`` limit by raising ``StopIteration`` once it has been reached. Parameters ---------- size : int, optional Number of rows to read in this chunk. Defaults to the ``chunksize`` passed when the reader was created. Returns ------- DataFrame The next chunk of parsed rows. Raises ------ StopIteration If ``nrows`` was set and has already been reached. See Also -------- TextFileReader.read : Read rows from the file into a DataFrame. read_csv : Read a comma-separated values (csv) file into a DataFrame. Examples -------- >>> with pd.read_csv("data.csv", iterator=True) as reader: # doctest: +SKIP ... first_five = reader.get_chunk(5) """ if size is None: size = self.chunksize if self.nrows is not None: if self._currow >= self.nrows: raise StopIteration if size is None: size = self.nrows - self._currow else: size = min(size, self.nrows - self._currow) return self.read(nrows=size) def __enter__(self) -> Self: return self def __exit__( self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, ) -> None: self.close() def TextParser(*args, **kwds) -> TextFileReader: """ Converts lists of lists/tuples into DataFrames with proper type inference and optional (e.g. string to datetime) conversion. Also enables iterating lazily over chunks of large files Parameters ---------- data : file-like object or list delimiter : separator character to use dialect : str or csv.Dialect class / instance, optional Ignored if delimiter is longer than 1 character names : sequence, default header : int, default 0 Row to use to parse column labels. Defaults to the first row. Prior rows will be discarded index_col : int or list, optional Column or columns to use as the (possibly hierarchical) index has_index_names: bool, default False True if the cols defined in index_col have an index name and are not in the header. na_values : scalar, str, list-like, or dict, optional Additional strings to recognize as NA/NaN. keep_default_na : bool, default True thousands : str, optional Thousands separator comment : str, optional Comment out remainder of line parse_dates : bool, default False date_format : str or dict of column -> format, default ``None`` .. versionadded:: 2.0.0 skiprows : list of integers Row numbers to skip skipfooter : int Number of line at bottom of file to skip converters : dict, optional Dict of functions for converting values in certain columns. Keys can either be integers or column labels, values are functions that take one input argument, the cell (not column) content, and return the transformed content. encoding : str, optional Encoding to use for UTF when reading/writing (ex. 'utf-8') float_precision : str, optional Specifies which converter the C engine should use for floating-point values. The options are `None` or `high` for the ordinary converter, `legacy` for the original lower precision pandas converter, and `round_trip` for the round-trip converter. .. deprecated:: 3.1.0 All float precision modes now use the same converter. The ``float_precision`` argument will be removed in a future version. """ kwds["engine"] = "python" return TextFileReader(*args, **kwds) def _clean_na_values(na_values, keep_default_na: bool = True, floatify: bool = True): na_fvalues: set | dict if na_values is None: if keep_default_na: na_values = STR_NA_VALUES else: na_values = set() na_fvalues = set() elif isinstance(na_values, dict): old_na_values = na_values.copy() na_values = {} # Prevent aliasing. # Convert the values in the na_values dictionary # into array-likes for further use. This is also # where we append the default NaN values, provided # that `keep_default_na=True`. for k, v in old_na_values.items(): if not is_list_like(v): v = [v] if keep_default_na: v = set(v) | STR_NA_VALUES na_values[k] = _stringify_na_values(v, floatify) na_fvalues = {k: _floatify_na_values(v) for k, v in na_values.items()} else: if not is_list_like(na_values): na_values = [na_values] na_values = _stringify_na_values(na_values, floatify) if keep_default_na: na_values = na_values | STR_NA_VALUES na_fvalues = _floatify_na_values(na_values) return na_values, na_fvalues def _floatify_na_values(na_values) -> set[float]: # create float versions of the na_values result = set() for v in na_values: try: v = float(v) if not np.isnan(v): result.add(v) except (TypeError, ValueError, OverflowError): pass return result def _stringify_na_values(na_values, floatify: bool) -> set[str | float]: """return a stringified and numeric for these values""" result: list[str | float] = [] for x in na_values: result.append(str(x)) result.append(x) try: v = float(x) # we are like 999 here if v == int(v): v = int(v) result.append(f"{v}.0") result.append(str(v)) if floatify: result.append(v) except (TypeError, ValueError, OverflowError): pass if floatify: try: result.append(int(x)) except (TypeError, ValueError, OverflowError): pass return set(result) def _refine_defaults_read( dialect: str | csv.Dialect | type[csv.Dialect] | None, delimiter: str | lib.NoDefault | None, engine: CSVEngine | None, sep: str | lib.NoDefault | None, on_bad_lines: str | Callable, names: Sequence[Hashable] | lib.NoDefault | None, defaults: dict[str, Any], dtype_backend: DtypeBackend | lib.NoDefault, lineterminator: str | bytes | None, ): """Validate/refine default values of input parameters of read_csv, read_table. Parameters ---------- dialect : str or csv.Dialect If provided, this parameter will override values (default or not) for the following parameters: `delimiter`, `doublequote`, `escapechar`, `skipinitialspace`, `quotechar`, and `quoting`. If it is necessary to override values, a ParserWarning will be issued. See csv.Dialect documentation for more details. delimiter : str or object Alias for sep, taking the same sentinel value when not provided. engine : {'c', 'python'} Parser engine to use. The C engine is faster while the python engine is currently more feature-complete. sep : str or object A delimiter provided by the user (str) or a sentinel value, i.e. pandas._libs.lib.no_default. on_bad_lines : str, callable An option for handling bad lines or a sentinel value(None). names : array-like, optional List of column names to use. If the file contains a header row, then you should explicitly pass ``header=0`` to override the column names. Duplicates in this list are not allowed. defaults: dict Default values of input parameters. lineterminator : str, bytes or None Line terminator passed by the user, if any. Returns ------- kwds : dict Input parameters with correct values. """ # fix types for sep, delimiter to Union(str, Any) delim_default = defaults["delimiter"] kwds: dict[str, Any] = {} # gh-23761 # # When a dialect is passed, it overrides any of the overlapping # parameters passed in directly. We don't want to warn if the # default parameters were passed in (since it probably means # that the user didn't pass them in explicitly in the first place). # # "delimiter" is the annoying corner case because we alias it to # "sep" before doing comparison to the dialect values later on. # Thus, we need a flag to indicate that we need to "override" # the comparison to dialect values by checking if default values # for BOTH "delimiter" and "sep" were provided. if dialect is not None: kwds["sep_override"] = delimiter is lib.no_default and ( sep is lib.no_default or sep == delim_default ) if delimiter is not lib.no_default and sep is not lib.no_default: raise ValueError("Specified a sep and a delimiter; you can only specify one.") kwds["names"] = None if names is lib.no_default else names # Alias sep -> delimiter. Both use lib.no_default rather than None as the # "not passed" sentinel, so an explicit None means sniff (GH#47024). if delimiter is lib.no_default: delimiter = sep if isinstance(lineterminator, (bytes, bytearray)): # the C engine accepts these; compare on the character it will use lineterminator = lineterminator.decode("latin-1") # GH#43528, GH#51801: the C engine silently mis-parses these, as the field # separator is consumed as a line terminator before it can split a field. # A custom lineterminator takes over that role, leaving "\n"/"\r" free to # separate fields. if delimiter in ("\n", "\r") and lineterminator in (None, delimiter): raise ValueError( f"Specified {delimiter!r} as separator or delimiter, but a line " f"terminator cannot be used as a separator. To parse {delimiter!r} " f"as a separator, pass a lineterminator other than {delimiter!r} " "(engine='c' only).", ) if delimiter is lib.no_default: # assign default separator value kwds["delimiter"] = delim_default else: kwds["delimiter"] = delimiter if engine is not None: kwds["engine_specified"] = True else: kwds["engine"] = "c" kwds["engine_specified"] = False if on_bad_lines == "error": kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.BLHM_ERROR elif on_bad_lines == "warn": kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.BLHM_WARN elif on_bad_lines == "skip": kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.BLHM_SKIP elif callable(on_bad_lines): if engine not in ["python", "pyarrow"]: raise ValueError( "on_bad_line can only be a callable function " "if engine='python' or 'pyarrow'" ) kwds["on_bad_lines"] = on_bad_lines else: raise ValueError(f"Argument {on_bad_lines} is invalid for on_bad_lines") check_dtype_backend(dtype_backend) kwds["dtype_backend"] = dtype_backend return kwds def _extract_dialect( kwds: dict[str, str | csv.Dialect | type[csv.Dialect]], ) -> csv.Dialect | type[csv.Dialect] | None: """ Extract concrete csv dialect instance. Returns ------- csv.Dialect or None """ if kwds.get("dialect") is None: return None dialect = kwds["dialect"] if isinstance(dialect, str) and dialect in csv.list_dialects(): # get_dialect is typed to return a `_csv.Dialect` for some reason in typeshed tdialect = cast("csv.Dialect", csv.get_dialect(dialect)) _validate_dialect(tdialect) else: _validate_dialect(dialect) tdialect = cast("csv.Dialect", dialect) return tdialect MANDATORY_DIALECT_ATTRS = ( "delimiter", "doublequote", "escapechar", "skipinitialspace", "quotechar", "quoting", ) def _validate_dialect(dialect: csv.Dialect | type[csv.Dialect] | str) -> None: """ Validate csv dialect instance. Raises ------ ValueError If incorrect dialect is provided. """ for param in MANDATORY_DIALECT_ATTRS: if not hasattr(dialect, param): raise ValueError(f"Invalid dialect {dialect} provided") def _merge_with_dialect_properties( dialect: csv.Dialect | type[csv.Dialect], defaults: dict[str, Any], ) -> dict[str, Any]: """ Merge default kwargs in TextFileReader with dialect parameters. Parameters ---------- dialect : csv.Dialect Concrete csv dialect. See csv.Dialect documentation for more details. defaults : dict Keyword arguments passed to TextFileReader. Returns ------- kwds : dict Updated keyword arguments, merged with dialect parameters. """ kwds = defaults.copy() for param in MANDATORY_DIALECT_ATTRS: dialect_val = getattr(dialect, param) parser_default = parser_defaults[param] provided = kwds.get(param, parser_default) # Messages for conflicting values between the dialect # instance and the actual parameters provided. conflict_msgs = [] # Don't warn if the default parameter was passed in, # even if it conflicts with the dialect (gh-23761). if provided not in (parser_default, dialect_val): msg = ( f"Conflicting values for '{param}': '{provided}' was " f"provided, but the dialect specifies '{dialect_val}'. " "Using the dialect-specified value." ) # Annoying corner case for not warning about # conflicts between dialect and delimiter parameter. # Refer to the outer "_read_" function for more info. if not (param == "delimiter" and kwds.pop("sep_override", False)): conflict_msgs.append(msg) if conflict_msgs: warnings.warn( "\n\n".join(conflict_msgs), ParserWarning, stacklevel=find_stack_level() ) kwds[param] = dialect_val return kwds def _validate_skipfooter(kwds: dict[str, Any]) -> None: """ Check whether skipfooter is compatible with other kwargs in TextFileReader. Parameters ---------- kwds : dict Keyword arguments passed to TextFileReader. Raises ------ ValueError If skipfooter is not compatible with other parameters. """ if kwds.get("skipfooter"): if kwds.get("iterator") or kwds.get("chunksize"): raise ValueError("'skipfooter' not supported for iteration") if kwds.get("nrows"): raise ValueError("'skipfooter' not supported with 'nrows'")