readstat/

formats.rs

//! SAS format string classification using regex-based detection.
//!
//! SAS variables carry format strings (e.g. `DATE9`, `DATETIME22.3`, `TIME8`) that
//! determine how raw numeric values should be interpreted. This module classifies
//! those format strings into [`ReadStatVarFormatClass`] variants (Date, `DateTime`,
//! Time, and their sub-second precision variants), enabling correct Arrow type mapping.
//!
//! Supports all 118+ SAS date/time/datetime formats including ISO 8601 variants,
//! national language (`NL*`) formats, and precision-based datetime/time formats.

use std::sync::LazyLock;

use regex::Regex;

use crate::rs_var::ReadStatVarFormatClass;

// DATETIME requesting nanosecond output (DATETIMEw.d where d=7-9). The SAS
// numeric may have less source precision, depending on the datetime magnitude.
static RE_DATETIME_WITH_NANO: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?xi)^DATETIME[0-9]{1,2}\.[7-9]$").unwrap());

// DATETIME with microsecond precision (DATETIMEw.d where d=4-6)
static RE_DATETIME_WITH_MICRO: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?xi)^DATETIME[0-9]{1,2}\.[4-6]$").unwrap());

// DATETIME with millisecond precision (DATETIMEw.d where d=1-3)
static RE_DATETIME_WITH_MILLI: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?xi)^DATETIME[0-9]{1,2}\.[1-3]$").unwrap());

// TIME with nanosecond precision (TIMEw.d where d=7-9)
static RE_TIME_WITH_NANO: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?xi)^TIME[0-9]{1,2}\.[7-9]$").unwrap());

// TIME with microsecond precision (TIMEw.d where d=4-6)
static RE_TIME_WITH_MICRO: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?xi)^TIME[0-9]{1,2}\.[4-6]$").unwrap());

// TIME with millisecond precision (TIMEw.d where d=1-3)
static RE_TIME_WITH_MILLI: LazyLock<Regex> =
    LazyLock::new(|| Regex::new(r"(?xi)^TIME[0-9]{1,2}\.[1-3]$").unwrap());

// All time formats - checked before datetime to catch NLDATMTM and NLDATMTZ
// Suffix allows numeric widths and decimal places (8, 8.2).
static RE_TIME: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(
        r"(?xi)
        ^(
            B8601LZ  |
            B8601TM  |
            B8601TX  |
            B8601TZ  |
            E8601LZ  |
            E8601TM  |
            E8601TX  |
            E8601TZ  |
            HHMM     |
            HOUR     |
            MMSS     |
            NLDATMTM |
            NLDATMTZ |
            NLTIMAP  |
            NLTIME   |
            TIMEAMPM |
            TIME     |
            TOD
        )[A-Z0-9]*(\.[A-Z0-9]*)?$",
    )
    .unwrap()
});

// All datetime formats - checked before date to catch DATEAMPM and DATETIME
// NLDATM matches all NLDATM* variants; NLDATMTM/NLDATMTZ already caught by RE_TIME
static RE_DATETIME: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(
        r"(?xi)
        ^(
            B8601DT  |
            B8601DX  |
            B8601DZ  |
            B8601LX  |
            DATEAMPM |
            DATETIME |
            E8601DT  |
            E8601DX  |
            E8601DZ  |
            E8601LX  |
            MDYAMPM  |
            NLDATM
        )[A-Z0-9]*(\.[A-Z0-9]*)?$",
    )
    .unwrap()
});

// All date formats
static RE_DATE: LazyLock<Regex> = LazyLock::new(|| {
    Regex::new(
        r"(?xi)
        ^(
            B8601DA   |
            B8601DN   |
            DATE      |
            DAY       |
            DDMMYY    |
            DOWNAME   |
            DTDATE    |
            DTMONYY   |
            DTWKDATX  |
            DTYEAR    |
            DTYYQC    |
            E8601DA   |
            E8601DN   |
            JULDAY    |
            JULIAN    |
            MMDDYY    |
            MMYY      |
            MONNAME   |
            MONTH     |
            MONYY     |
            NENGO     |
            NLDATE    |
            QTRR?     |
            WEEKDATX  |
            WEEKDAY   |
            YEAR      |
            YYMMDD    |
            YYMM      |
            YYMON     |
            YYQR      |
            YYQ       |
            YYWEEK[UVW]
        )[A-Z0-9]*(\.[A-Z0-9]*)?$",
    )
    .unwrap()
});

/// Classifies a SAS format string into a [`ReadStatVarFormatClass`].
///
/// Returns `Some(class)` for recognized date/time/datetime formats, or `None`
/// for numeric/character formats that don't represent temporal data.
/// Matching is case-insensitive and handles bare names (`DATE`) and numeric
/// widths (`DATE9`).
pub(crate) fn match_var_format(v: &str) -> Option<ReadStatVarFormatClass> {
    // Check order matters:
    // 1. DATETIME precision variants (most specific, numeric width only)
    // 2. TIME precision variants (most specific, numeric width only)
    // 3. Time (catches NLDATMTM, NLDATMTZ before general NLDATM datetime match)
    // 4. General datetime (catches DATEAMPM, DATETIME before DATE match)
    // 5. Date (everything else)
    if RE_DATETIME_WITH_NANO.is_match(v) {
        Some(ReadStatVarFormatClass::DateTimeWithNanoseconds)
    } else if RE_DATETIME_WITH_MICRO.is_match(v) {
        Some(ReadStatVarFormatClass::DateTimeWithMicroseconds)
    } else if RE_DATETIME_WITH_MILLI.is_match(v) {
        Some(ReadStatVarFormatClass::DateTimeWithMilliseconds)
    } else if RE_TIME_WITH_NANO.is_match(v) {
        Some(ReadStatVarFormatClass::TimeWithNanoseconds)
    } else if RE_TIME_WITH_MICRO.is_match(v) {
        Some(ReadStatVarFormatClass::TimeWithMicroseconds)
    } else if RE_TIME_WITH_MILLI.is_match(v) {
        Some(ReadStatVarFormatClass::TimeWithMilliseconds)
    } else if RE_TIME.is_match(v) {
        Some(ReadStatVarFormatClass::Time)
    } else if RE_DATETIME.is_match(v) {
        Some(ReadStatVarFormatClass::DateTime)
    } else if RE_DATE.is_match(v) {
        Some(ReadStatVarFormatClass::Date)
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // --- Date formats ---

    #[test]
    fn date_formats_with_numeric_width() {
        // Existing formats that were already supported
        assert_eq!(
            match_var_format("DATE9"),
            Some(ReadStatVarFormatClass::Date)
        );
        assert_eq!(
            match_var_format("DDMMYY10"),
            Some(ReadStatVarFormatClass::Date)
        );
        assert_eq!(
            match_var_format("DDMMYYB10"),
            Some(ReadStatVarFormatClass::Date)
        );
        assert_eq!(
            match_var_format("MMDDYY10"),
            Some(ReadStatVarFormatClass::Date)
        );
        assert_eq!(
            match_var_format("YYMMDD10"),
            Some(ReadStatVarFormatClass::Date)
        );
    }

    #[test]
    fn recognized_date_format_names() {
        let date_formats = [
            "B8601DA",
            "B8601DN",
            "DATE",
            "DAY",
            "DDMMYY",
            "DDMMYYD",
            "DOWNAME",
            "DTDATE",
            "DTMONYY",
            "DTWKDATX",
            "DTYEAR",
            "DTYYQC",
            "E8601DA",
            "E8601DN",
            "JULDAY",
            "JULIAN",
            "MMDDYY",
            "MMDDYYD",
            "MMYY",
            "MMYYD",
            "MONNAME",
            "MONTH",
            "MONYY",
            "NENGO",
            "NLDATE",
            "NLDATECP",
            "NLDATEL",
            "NLDATEM",
            "NLDATEMD",
            "NLDATEMDL",
            "NLDATEMDM",
            "NLDATEMDS",
            "NLDATEMN",
            "NLDATES",
            "NLDATEW",
            "NLDATEWN",
            "NLDATEYM",
            "NLDATEYML",
            "NLDATEYMM",
            "NLDATEYMS",
            "NLDATEYQ",
            "NLDATEYQL",
            "NLDATEYQM",
            "NLDATEYQS",
            "NLDATEYR",
            "NLDATEYW",
            "QTR",
            "QTRR",
            "WEEKDATX",
            "WEEKDAY",
            "YEAR",
            "YYMM",
            "YYMMDD",
            "YYMMDDD",
            "YYMMD",
            "YYMON",
            "YYQ",
            "YYQD",
            "YYQR",
            "YYQRD",
            "YYWEEKU",
            "YYWEEKV",
            "YYWEEKW",
        ];
        for fmt in &date_formats {
            assert_eq!(
                match_var_format(fmt),
                Some(ReadStatVarFormatClass::Date),
                "Expected Date for format: {fmt}"
            );
        }
    }

    // --- Time formats ---

    #[test]
    fn time_format_bare() {
        assert_eq!(match_var_format("TIME"), Some(ReadStatVarFormatClass::Time));
        assert_eq!(
            match_var_format("TIME8"),
            Some(ReadStatVarFormatClass::Time)
        );
    }

    #[test]
    fn recognized_time_format_names() {
        let time_formats = [
            "B8601LZ", "B8601TM", "B8601TX", "B8601TZ", "E8601LZ", "E8601TM", "E8601TX", "E8601TZ",
            "HHMM", "HOUR", "MMSS", "NLDATMTM", "NLDATMTZ", "NLTIMAP", "NLTIME", "TIME",
            "TIMEAMPM", "TOD",
        ];
        for fmt in &time_formats {
            assert_eq!(
                match_var_format(fmt),
                Some(ReadStatVarFormatClass::Time),
                "Expected Time for format: {fmt}"
            );
        }
    }

    // --- Datetime formats ---

    #[test]
    fn datetime_format_with_numeric_width() {
        assert_eq!(
            match_var_format("DATETIME22"),
            Some(ReadStatVarFormatClass::DateTime)
        );
    }

    #[test]
    fn time_precision_formats() {
        // .1-3 → milliseconds
        for fmt in ["TIME10.1", "TIME11.2", "TIME12.3"] {
            assert_eq!(
                match_var_format(fmt),
                Some(ReadStatVarFormatClass::TimeWithMilliseconds),
                "Expected milliseconds for {fmt}"
            );
        }
        // .4-6 → microseconds
        for fmt in ["TIME13.4", "TIME14.5", "TIME15.6"] {
            assert_eq!(
                match_var_format(fmt),
                Some(ReadStatVarFormatClass::TimeWithMicroseconds),
                "Expected microseconds for {fmt}"
            );
        }
        // .7-9 → nanoseconds
        for fmt in ["TIME16.7", "TIME17.8", "TIME18.9"] {
            assert_eq!(
                match_var_format(fmt),
                Some(ReadStatVarFormatClass::TimeWithNanoseconds),
                "Expected nanoseconds for {fmt}"
            );
        }
        // Without precision decimal, should be plain Time
        assert_eq!(
            match_var_format("TIME15"),
            Some(ReadStatVarFormatClass::Time)
        );
    }

    #[test]
    fn datetime_precision_formats() {
        assert_eq!(
            match_var_format("DATETIME22.3"),
            Some(ReadStatVarFormatClass::DateTimeWithMilliseconds)
        );
        assert_eq!(
            match_var_format("DATETIME25.6"),
            Some(ReadStatVarFormatClass::DateTimeWithMicroseconds)
        );
        assert_eq!(
            match_var_format("DATETIME28.9"),
            Some(ReadStatVarFormatClass::DateTimeWithNanoseconds)
        );
    }

    #[test]
    fn recognized_datetime_format_names() {
        let datetime_formats = [
            "B8601DT",
            "B8601DX",
            "B8601DZ",
            "B8601LX",
            "DATEAMPM",
            "DATETIME",
            "E8601DT",
            "E8601DX",
            "E8601DZ",
            "E8601LX",
            "MDYAMPM",
            "NLDATM",
            "NLDATMAP",
            "NLDATMCP",
            "NLDATMDT",
            "NLDATML",
            "NLDATMM",
            "NLDATMMD",
            "NLDATMMDL",
            "NLDATMMDM",
            "NLDATMMDS",
            "NLDATMMN",
            "NLDATMS",
            "NLDATMW",
            "NLDATMWN",
            "NLDATMWZ",
            "NLDATMYM",
            "NLDATMYML",
            "NLDATMYMM",
            "NLDATMYMS",
            "NLDATMYQ",
            "NLDATMYQL",
            "NLDATMYQM",
            "NLDATMYQS",
            "NLDATMYR",
            "NLDATMYW",
            "NLDATMZ",
        ];
        for fmt in &datetime_formats {
            assert_eq!(
                match_var_format(fmt),
                Some(ReadStatVarFormatClass::DateTime),
                "Expected DateTime for format: {fmt}"
            );
        }
    }

    // --- Non-matching formats ---

    #[test]
    fn non_date_time_formats() {
        assert_eq!(match_var_format("BEST12"), None);
        assert_eq!(match_var_format("$30"), None);
        assert_eq!(match_var_format("$10"), None);
        assert_eq!(match_var_format("COMMA12"), None);
        assert_eq!(match_var_format(""), None);
    }

    // --- Case insensitivity ---

    #[test]
    fn case_insensitive() {
        assert_eq!(
            match_var_format("date9"),
            Some(ReadStatVarFormatClass::Date)
        );
        assert_eq!(
            match_var_format("datetime22"),
            Some(ReadStatVarFormatClass::DateTime)
        );
        assert_eq!(
            match_var_format("time8"),
            Some(ReadStatVarFormatClass::Time)
        );
    }

    // --- Property-based tests ---

    mod property_tests {
        use super::*;
        use proptest::prelude::*;
        use proptest::sample;

        proptest! {
            /// Arbitrary strings never cause a panic.
            #[test]
            fn arbitrary_strings_never_panic(s in "\\PC*") {
                let _ = match_var_format(&s);
            }

            /// Result is deterministic: same input always produces the same output.
            #[test]
            fn result_is_deterministic(s in "\\PC*") {
                let a = match_var_format(&s);
                let b = match_var_format(&s);
                prop_assert_eq!(a, b);
            }

            /// Case insensitivity: format classification is the same regardless of case.
            #[test]
            fn case_insensitive_matching(s in "[A-Za-z0-9.]{1,20}") {
                let upper = match_var_format(&s.to_uppercase());
                let lower = match_var_format(&s.to_lowercase());
                prop_assert_eq!(upper, lower, "case mismatch for '{}'", s);
            }

            /// Known date prefix + numeric width always classifies as Date.
            #[test]
            fn date_prefix_always_matches(
                prefix in sample::select(vec![
                    "DATE", "DDMMYY", "MMDDYY", "YYMMDD", "JULIAN", "MONYY",
                    "YEAR", "MONTH", "DAY", "WEEKDAY", "JULDAY",
                ]),
                width in 1u32..30
            ) {
                let fmt = format!("{prefix}{width}");
                prop_assert_eq!(
                    match_var_format(&fmt),
                    Some(ReadStatVarFormatClass::Date),
                    "Expected Date for '{}'", fmt
                );
            }

            /// Known time prefix + numeric width always classifies as Time.
            #[test]
            fn time_prefix_always_matches(
                prefix in sample::select(vec![
                    "TIME", "HHMM", "HOUR", "MMSS", "TOD", "TIMEAMPM",
                ]),
                width in 1u32..30
            ) {
                let fmt = format!("{prefix}{width}");
                prop_assert_eq!(
                    match_var_format(&fmt),
                    Some(ReadStatVarFormatClass::Time),
                    "Expected Time for '{}'", fmt
                );
            }

            /// DATETIME + width (no decimal) always classifies as DateTime.
            #[test]
            fn datetime_prefix_always_matches(width in 1u32..30) {
                let fmt = format!("DATETIME{width}");
                prop_assert_eq!(
                    match_var_format(&fmt),
                    Some(ReadStatVarFormatClass::DateTime),
                    "Expected DateTime for '{}'", fmt
                );
            }

            /// DATETIME with precision 1-3 → milliseconds, 4-6 → microseconds, 7-9 → nanoseconds.
            #[test]
            fn datetime_precision_classifies_correctly(
                width in 1u32..30,
                precision in 1u32..=9
            ) {
                let fmt = format!("DATETIME{width}.{precision}");
                let expected = match precision {
                    1..=3 => ReadStatVarFormatClass::DateTimeWithMilliseconds,
                    4..=6 => ReadStatVarFormatClass::DateTimeWithMicroseconds,
                    7..=9 => ReadStatVarFormatClass::DateTimeWithNanoseconds,
                    _ => unreachable!(),
                };
                prop_assert_eq!(
                    match_var_format(&fmt),
                    Some(expected),
                    "Wrong class for '{}'", fmt
                );
            }

            /// TIME with precision 1-3 → milliseconds, 4-6 → microseconds, 7-9 → nanoseconds.
            #[test]
            fn time_precision_classifies_correctly(
                width in 1u32..30,
                precision in 1u32..=9
            ) {
                let fmt = format!("TIME{width}.{precision}");
                let expected = match precision {
                    1..=3 => ReadStatVarFormatClass::TimeWithMilliseconds,
                    4..=6 => ReadStatVarFormatClass::TimeWithMicroseconds,
                    7..=9 => ReadStatVarFormatClass::TimeWithNanoseconds,
                    _ => unreachable!(),
                };
                prop_assert_eq!(
                    match_var_format(&fmt),
                    Some(expected),
                    "Wrong class for '{}'", fmt
                );
            }

            /// Numeric-only formats (BEST, COMMA, etc.) never match as temporal.
            #[test]
            fn numeric_formats_return_none(
                prefix in sample::select(vec!["BEST", "COMMA", "DOLLAR", "PERCENT", "F", "E"]),
                width in 1u32..30
            ) {
                let fmt = format!("{prefix}{width}");
                prop_assert_eq!(
                    match_var_format(&fmt),
                    None,
                    "Expected None for '{}'", fmt
                );
            }
        }
    }
}