readstat/

lib.rs

//! Read SAS binary files (`.sas7bdat`) and convert them to other formats.
//!
//! This crate provides a library for parsing SAS binary data files using FFI
//! bindings to the [ReadStat](https://github.com/WizardMac/ReadStat) C library,
//! then converting the parsed data into Apache Arrow [`RecordBatch`](arrow_array::RecordBatch)
//! format for output as CSV, Feather (Arrow IPC), NDJSON, or Parquet.
//!
//! **Note:** While the underlying [`readstat-sys`](https://docs.rs/readstat-sys) crate
//! exposes bindings for all formats supported by `ReadStat` (SAS, SPSS, Stata),
//! this crate only implements parsing and conversion for **SAS `.sas7bdat` files**.
//! SPSS and Stata support is a possible future addition, but is **not planned at
//! this time** — if you need those formats today, the `readstat-sys` bindings
//! already expose the complete SPSS (`.sav`, `.zsav`, `.por`) and Stata (`.dta`)
//! C API to build on.
//!
//! # Data Pipeline
//!
//! ```text
//! .sas7bdat file
//!     → ReadStat C library (FFI parsing via callbacks)
//!         → Typed Arrow builders (StringBuilder, Float64Builder, etc.)
//!             → Arrow RecordBatch
//!                 → Output format (CSV / Feather / NDJSON / Parquet)
//! ```
//!
//! # Examples
//!
//! ## Quick start
//!
//! [`ReadStatReader`] is the primary API. It reads paths, owned bytes, or memory
//! maps and supports metadata, row/column selection, collection, and bounded
//! chunk callbacks. The two free functions are shorthand for path defaults:
//!
//! ```no_run
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let md = readstat::read_metadata("data.sas7bdat")?;
//! println!("{:?} rows x {} columns", md.row_count, md.var_count);
//!
//! let batch = readstat::read_to_batch("data.sas7bdat")?;
//! println!("Schema: {:?}", batch.schema());
//! # Ok(())
//! # }
//! ```
//!
//! For bounded-memory streaming and selection, configure a reader:
//!
//! ```no_run
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let reader = readstat::ReadStatReader::from_path("data.sas7bdat")?
//!     .columns(["name", "age"])
//!     .rows(100, Some(1_000))
//!     .chunk_rows(250);
//! reader.visit(|batch| {
//!     println!("received {} rows", batch.num_rows());
//!     Ok(())
//! })?;
//! # Ok(())
//! # }
//! ```
//!
//! The lower-level metadata and data types remain public for compatibility and
//! specialized integrations, but new callers should start with [`ReadStatReader`].
//!
//! ## Inspect file metadata
//!
//! Read metadata without loading any row data. Useful for discovering
//! schema, row counts, variable types, and SAS format classifications.
//!
//! ```no_run
//! use readstat::{ReadStatPath, ReadStatMetadata};
//!
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let rsp = ReadStatPath::new("data.sas7bdat")?;
//!
//! let mut md = ReadStatMetadata::new();
//! md.read_metadata(&rsp, false)?;
//!
//! println!("Rows: {:?}, Variables: {}", md.row_count, md.var_count);
//! println!("Encoding: {}", md.file_encoding);
//! println!("Compression: {:?}", md.compression);
//!
//! // Iterate over variable metadata
//! for (idx, var) in &md.vars {
//!     println!(
//!         "  [{idx}] {} ({:?}, format: {})",
//!         var.var_name, var.var_type_class, var.var_format
//!     );
//! }
//!
//! // The Arrow schema is also available
//! println!("Schema: {:?}", md.schema);
//! # Ok(())
//! # }
//! ```
//!
//! ## Read all data into Arrow `RecordBatch`
//!
//! Parse the entire file into a single Arrow [`RecordBatch`](arrow_array::RecordBatch).
//! Best for smaller files that fit comfortably in memory.
//!
//! ```no_run
//! use readstat::{ReadStatPath, ReadStatMetadata, ReadStatData};
//!
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let rsp = ReadStatPath::new("data.sas7bdat")?;
//!
//! // Read metadata first
//! let mut md = ReadStatMetadata::new();
//! md.read_metadata(&rsp, false)?;
//!
//! // Read all rows into a single chunk
//! let row_count = md.row_count.ok_or(readstat::ReadStatError::RowCountUnavailable)? as u32;
//! let mut d = ReadStatData::new().init(md, 0, row_count);
//! d.read_data(&rsp)?;
//!
//! // Access the Arrow RecordBatch
//! if let Some(batch) = &d.batch {
//!     println!("Got {} rows x {} columns", batch.num_rows(), batch.num_columns());
//!     println!("Schema: {:?}", batch.schema());
//! }
//! # Ok(())
//! # }
//! ```
//!
//! ## Stream data in chunks and write to Parquet
//!
//! For large files, read in streaming chunks to control memory usage.
//! Each chunk is parsed and written incrementally.
//!
//! ```no_run
//! use readstat::{OutFormat, ReadStatReader, ReadStatWriter, WriteConfig};
//!
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let reader = ReadStatReader::from_path("data.sas7bdat")?.chunk_rows(10_000);
//! let schema = std::sync::Arc::new(reader.metadata()?.schema);
//! let config = WriteConfig::new(OutFormat::Parquet)
//!     .output("output.parquet")?
//!     .overwrite(false);
//! let mut writer = ReadStatWriter::new(config, schema)?;
//! reader.visit(|batch| writer.write(&batch))?;
//! println!("wrote {} rows", writer.finish()?);
//! # Ok(())
//! # }
//! ```
//!
//! ## Read from in-memory bytes
//!
//! Parse a `.sas7bdat` file from a byte slice instead of the filesystem.
//! Useful for cloud storage, HTTP uploads, WASM targets, and testing.
//!
//! ```no_run
//! use readstat::{ReadStatMetadata, ReadStatData};
//!
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! # let sas_bytes: &[u8] = &[];
//! // sas_bytes: &[u8] — obtained from S3, HTTP, etc.
//! let mut md = ReadStatMetadata::new();
//! md.read_metadata_from_bytes(sas_bytes, false)?;
//!
//! let row_count = md.row_count.ok_or(readstat::ReadStatError::RowCountUnavailable)? as u32;
//! let mut d = ReadStatData::new().init(md, 0, row_count);
//! d.read_data_from_bytes(sas_bytes)?;
//!
//! if let Some(batch) = &d.batch {
//!     println!("Parsed {} rows from bytes", batch.num_rows());
//! }
//! # Ok(())
//! # }
//! ```
//!
//! ## Filter to specific columns
//!
//! Select only specific columns before reading data. Unselected columns
//! are skipped during parsing, reducing both memory and CPU usage.
//!
//! ```no_run
//! use readstat::{ReadStatPath, ReadStatMetadata, ReadStatData};
//!
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let rsp = ReadStatPath::new("data.sas7bdat")?;
//!
//! let mut md = ReadStatMetadata::new();
//! md.read_metadata(&rsp, false)?;
//!
//! // Select only these columns
//! let columns = vec!["name".to_string(), "age".to_string()];
//!
//! if let Some(mapping) = md.resolve_selected_columns(Some(columns))? {
//!     // `init_filtered` applies the filter and initializes in the correct
//!     // order — pass the *original* metadata; filtering happens internally.
//!     let row_count = u32::try_from(md.row_count.ok_or(readstat::ReadStatError::RowCountUnavailable)?)?;
//!     let mut d = ReadStatData::new().init_filtered(md, &mapping, 0, row_count);
//!     d.read_data(&rsp)?;
//!
//!     if let Some(batch) = &d.batch {
//!         // batch only contains "name" and "age" columns
//!         println!(
//!             "Columns: {:?}",
//!             batch.schema().fields().iter().map(|f| f.name()).collect::<Vec<_>>()
//!         );
//!     }
//! }
//! # Ok(())
//! # }
//! ```
//!
//! ## Convert `RecordBatch` to in-memory bytes
//!
//! Serialize a parsed [`RecordBatch`](arrow_array::RecordBatch) directly to
//! in-memory bytes without writing to a file. Useful for HTTP responses,
//! message queues, or piping to other Arrow-aware tools.
//!
//! ```no_run
//! use readstat::{ReadStatPath, ReadStatMetadata, ReadStatData};
//! # #[cfg(feature = "parquet")]
//! use readstat::write_batch_to_parquet_bytes;
//! # #[cfg(feature = "csv")]
//! use readstat::write_batch_to_csv_bytes;
//!
//! # fn main() -> Result<(), readstat::ReadStatError> {
//! let rsp = ReadStatPath::new("data.sas7bdat")?;
//!
//! let mut md = ReadStatMetadata::new();
//! md.read_metadata(&rsp, false)?;
//!
//! let row_count = md.row_count.ok_or(readstat::ReadStatError::RowCountUnavailable)? as u32;
//! let mut d = ReadStatData::new().init(md, 0, row_count);
//! d.read_data(&rsp)?;
//!
//! if let Some(batch) = &d.batch {
//!     // Get Parquet bytes (e.g. for an HTTP response)
//!     # #[cfg(feature = "parquet")]
//!     let parquet_bytes = write_batch_to_parquet_bytes(batch)?;
//!
//!     // Or CSV bytes
//!     # #[cfg(feature = "csv")]
//!     let csv_bytes = write_batch_to_csv_bytes(batch)?;
//! }
//! # Ok(())
//! # }
//! ```
//!
//! # Key Functions
//!
//! - [`read_metadata`] — Read file/variable metadata in one call
//! - [`read_to_batch`] — Read an entire file into one Arrow [`RecordBatch`](arrow_array::RecordBatch)
//!
//! # Key Types
//!
//! - [`ReadStatPath`] — Validated input file path for SAS files
//! - [`WriteConfig`] — Output configuration (path, format, compression)
//! - [`ReadStatMetadata`] — File-level metadata (row/var counts, encoding, Arrow schema)
//! - [`ReadStatData`] — Parsed row data, convertible to Arrow [`RecordBatch`](arrow_array::RecordBatch)
//! - [`ReadStatVarFormatClass`] — SAS format classification (Date, `DateTime`, Time variants)
//! - [`ReadStatWriter`] — Writes Arrow batches to the configured output format
//!
//! # Features
//!
//! Output format writers are feature-gated (all enabled by default):
//!
//! | Feature | Format | Notes |
//! |---------|--------|-------|
//! | `csv` | CSV | Comma-separated values via `arrow-csv` |
//! | `parquet` | Parquet | Columnar format via `parquet` crate, 5 compression codecs |
//! | `feather` | Feather | Arrow IPC format via `arrow-ipc` |
//! | `ndjson` | NDJSON | Newline-delimited JSON via `arrow-json` |
//! | `sql` | SQL | Query data with SQL via DataFusion (enabled by default) |
//!
//! # Arrow version policy
//!
//! This crate exposes Apache Arrow types ([`RecordBatch`](arrow_array::RecordBatch),
//! [`Schema`](arrow_schema::Schema), …) in its public API. To avoid the
//! "expected `RecordBatch`, found `RecordBatch`" mismatch that occurs when a
//! consumer pins a different Arrow version, the [`arrow`], [`arrow_array`], and
//! [`arrow_schema`] crates are re-exported here — prefer
//! `readstat::arrow_array::RecordBatch` over a direct Arrow dependency.
//!
//! Arrow is currently pinned to the **v58** ecosystem. Because Arrow types
//! appear in the public API, a major Arrow bump is a breaking change for this
//! crate and will come with a corresponding semver-major release.

#![cfg_attr(docsrs, feature(doc_cfg))]
#![warn(missing_docs)]
#![allow(clippy::module_name_repetitions)]
#![allow(clippy::must_use_candidate)]
#![allow(clippy::return_self_not_must_use)]

// Re-export the Arrow ecosystem crates so consumers can name the types that
// appear in this crate's public API without pinning Arrow themselves. See the
// "Arrow version policy" section above.
pub use api::{ReadStatReader, read_metadata, read_to_batch};
pub use arrow;
pub use arrow_array;
pub use arrow_schema;
pub use common::build_offsets;
pub use err::{ReadStatCError, ReadStatError};
pub use progress::ProgressCallback;
pub use rs_data::ReadStatData;
pub use rs_metadata::{ReadStatCompress, ReadStatEndian, ReadStatMetadata, ReadStatVarMetadata};
pub use rs_path::ReadStatPath;
#[cfg(feature = "sql")]
#[cfg_attr(docsrs, doc(cfg(feature = "sql")))]
pub use rs_query::{
    AsyncRecordBatchReceiver, AsyncRecordBatchSender, RecordBatchReceiver, RecordBatchSender,
    async_record_batch_channel, execute_sql, execute_sql_and_write_stream,
    execute_sql_and_write_stream_async, execute_sql_async, execute_sql_stream,
    execute_sql_stream_async, read_sql_file, record_batch_channel,
};
pub use rs_var::{ReadStatVarFormatClass, ReadStatVarType, ReadStatVarTypeClass};
#[cfg(feature = "parquet")]
pub use rs_write::ParallelParquetWriter;
#[cfg(all(any(feature = "csv", feature = "ndjson"), not(target_arch = "wasm32")))]
pub use rs_write::ParallelTextWriter;
pub use rs_write::ReadStatWriter;
#[cfg(feature = "csv")]
#[cfg_attr(docsrs, doc(cfg(feature = "csv")))]
pub use rs_write::write_batch_to_csv_bytes;
#[cfg(feature = "feather")]
#[cfg_attr(docsrs, doc(cfg(feature = "feather")))]
pub use rs_write::write_batch_to_feather_bytes;
#[cfg(feature = "ndjson")]
#[cfg_attr(docsrs, doc(cfg(feature = "ndjson")))]
pub use rs_write::write_batch_to_ndjson_bytes;
#[cfg(feature = "parquet")]
#[cfg_attr(docsrs, doc(cfg(feature = "parquet")))]
pub use rs_write::write_batch_to_parquet_bytes;
pub use rs_write_config::{OutFormat, ParquetCompression, WriteConfig};

mod api;
mod cb;
mod common;
mod err;
mod formats;
mod progress;
mod rs_buffer_io;
mod rs_data;
mod rs_metadata;
mod rs_parser;
mod rs_path;
#[cfg(feature = "sql")]
mod rs_query;
mod rs_var;
mod rs_write;
mod rs_write_config;