Skip to content

API: preprocessing.outliers

skyulf.preprocessing.outliers

Outlier-handling nodes package.

Split from a single 539-LOC module into per-node files

_common.py — shared mask helpers (y filter, pandas mask apply) iqr.py — IQR zscore.py — ZScore winsorize.py — Winsorize manual_bounds.py — ManualBounds elliptic.py — EllipticEnvelope clip_values.py — ClipValues (fixed bounds, clips instead of dropping rows)

All public names are re-exported here so existing imports such as from skyulf.preprocessing.outliers import IQRCalculator continue to work.

ClipValuesApplier

Bases: BaseApplier

Set values below lower to lower and above upper to upper.

Nulls stay null, other columns are untouched and every row is kept, so unlike ManualBounds this step also runs on scoring rows.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
class ClipValuesApplier(BaseApplier):
    """Set values below ``lower`` to ``lower`` and above ``upper`` to ``upper``.

    Nulls stay null, other columns are untouched and every row is kept, so
    unlike ManualBounds this step also runs on scoring rows.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return _clip_state(raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        params = _normalize(raw)
        params.pop("target_column", None)
        params.pop("_display_name", None)
        params.pop("_auto_columns", None)
        _fields(params, {"bounds"})
        bounds = _bounds(params["bounds"])
        if bounds != state["bounds"]:
            raise ValueError("Configured bounds disagree with fitted clipping bounds.")
        return {"bounds": bounds}

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Clip the configured columns on the active engine; ``y`` passes through."""
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        exprs = [
            _clip_polars_column(X, column, bound)
            for column, bound in params.get("bounds", {}).items()
            if column in X.columns
        ]
        return X.with_columns(exprs), y

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        out = X.copy()
        for column, bound in params.get("bounds", {}).items():
            if column not in out.columns:
                continue
            series = out[column]
            if is_decimal_series(series) or _fractional_bound(bound):
                series = pd.to_numeric(series).astype("float64")
            out[column] = series.clip(lower=bound.get("lower"), upper=bound.get("upper"))
        return out, y

apply(X, y, params)

Clip the configured columns on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
109
110
111
112
113
114
115
@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Clip the configured columns on the active engine; ``y`` passes through."""
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
 96
 97
 98
 99
100
101
102
103
104
105
106
107
@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    params = _normalize(raw)
    params.pop("target_column", None)
    params.pop("_display_name", None)
    params.pop("_auto_columns", None)
    _fields(params, {"bounds"})
    bounds = _bounds(params["bounds"])
    if bounds != state["bounds"]:
        raise ValueError("Configured bounds disagree with fitted clipping bounds.")
    return {"bounds": bounds}

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
91
92
93
94
@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return _clip_state(raw)

ClipValuesCalculator

Bases: BaseCalculator

Validate the configured bounds; nothing is learned from the data.

bounds maps a column to {"lower": ..., "upper": ...}; either side may be omitted. Use Winsorize for bounds learned from percentiles.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
@NodeRegistry.register(
    "ClipValues",
    ClipValuesApplier,
    execution_capabilities=(
        ExecutionCapability("pandas", "apply", "python_batch", "preserve", "row"),
        ExecutionCapability("polars", "apply", "local", "preserve", "row"),
    ),
)
@node_meta(
    id="ClipValues",
    name="Clip Values",
    category="Preprocessing",
    description=(
        "Limits numeric columns to fixed lower/upper bounds (e.g. employees at most 500) "
        "without removing rows."
    ),
    params={"bounds": {}},
    learns_from_data=False,
)
class ClipValuesCalculator(BaseCalculator):
    """Validate the configured bounds; nothing is learned from the data.

    ``bounds`` maps a column to ``{"lower": ..., "upper": ...}``; either side
    may be omitted. Use Winsorize for bounds learned from percentiles.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Promote fractionally bounded columns to match both runtime engines."""
        schema = input_schema
        for column, bound in (config.get("bounds") or {}).items():
            if column in schema.columns and _fractional_bound(bound):
                schema = schema.with_dtype(column, "float64")
        return schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ClipValuesArtifact:  # pylint: disable=arguments-differ
        """Return the validated bounds as the artifact."""
        raw = config.get("bounds") or {}
        bounds = {column: _clean_bound(column, bound) for column, bound in raw.items()}
        if bounds and X is not None:
            _check_columns(X.to_native() if hasattr(X, "to_native") else X, bounds)
        return {"type": "clip_values", "bounds": bounds}

fit(X, _y, config)

Return the validated bounds as the artifact.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
175
176
177
178
179
180
181
182
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ClipValuesArtifact:  # pylint: disable=arguments-differ
    """Return the validated bounds as the artifact."""
    raw = config.get("bounds") or {}
    bounds = {column: _clean_bound(column, bound) for column, bound in raw.items()}
    if bounds and X is not None:
        _check_columns(X.to_native() if hasattr(X, "to_native") else X, bounds)
    return {"type": "clip_values", "bounds": bounds}

infer_output_schema(input_schema, config)

Promote fractionally bounded columns to match both runtime engines.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py
165
166
167
168
169
170
171
172
173
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Promote fractionally bounded columns to match both runtime engines."""
    schema = input_schema
    for column, bound in (config.get("bounds") or {}).items():
        if column in schema.columns and _fractional_bound(bound):
            schema = schema.with_dtype(column, "float64")
    return schema

EllipticEnvelopeApplier

Bases: BaseApplier

Drop rows that any fitted per-column EllipticEnvelope model flags as an outlier.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
class EllipticEnvelopeApplier(BaseApplier):
    """Drop rows that any fitted per-column EllipticEnvelope model flags as an outlier."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved envelopes and skipped-column metadata without estimating covariance."""
        if not local_state_fields(
            raw,
            "elliptic_envelope",
            {"type", "models", "contamination", "warnings"},
            allow_empty=True,
        ):
            return raw
        models = raw["models"]
        if type(models) is not dict:
            raise ValueError("Fitted EllipticEnvelope models must be a dictionary.")
        for model in models.values():
            _validate_elliptic_model(model)
        validate_detector_warnings(raw["warnings"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe fitted row filtering with nonfinite rows preserved by the native applier."""
        if engine not in ("pandas", "polars"):
            return None
        EllipticEnvelopeApplier.validate_inference_state(state)
        effect = "filter" if state.get("models") else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the fitted per-column models on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        models = params.get("models", {})
        if not models:
            return X, y

        mask = pl.Series(_elliptic_mask_numpy(X, models))
        return X.filter(mask), _filter_y_polars(y, mask)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        models = params.get("models", {})
        if not models:
            return X, y
        mask = _elliptic_filter_pandas(X, models)
        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the fitted per-column models on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
161
162
163
164
165
166
167
168
169
170
171
@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the fitted per-column models on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe fitted row filtering with nonfinite rows preserved by the native applier.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
152
153
154
155
156
157
158
159
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe fitted row filtering with nonfinite rows preserved by the native applier."""
    if engine not in ("pandas", "polars"):
        return None
    EllipticEnvelopeApplier.validate_inference_state(state)
    effect = "filter" if state.get("models") else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "row")

validate_inference_state(raw) staticmethod

Inspect saved envelopes and skipped-column metadata without estimating covariance.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved envelopes and skipped-column metadata without estimating covariance."""
    if not local_state_fields(
        raw,
        "elliptic_envelope",
        {"type", "models", "contamination", "warnings"},
        allow_empty=True,
    ):
        return raw
    models = raw["models"]
    if type(models) is not dict:
        raise ValueError("Fitted EllipticEnvelope models must be a dictionary.")
    for model in models.values():
        _validate_elliptic_model(model)
    validate_detector_warnings(raw["warnings"])
    return raw

EllipticEnvelopeCalculator

Bases: BaseCalculator

Fit one univariate EllipticEnvelope per numeric column at the configured contamination.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
@NodeRegistry.register("EllipticEnvelope", EllipticEnvelopeApplier)
@node_meta(
    id="EllipticEnvelope",
    name="Elliptic Envelope",
    category="Preprocessing",
    description="Detect outliers in a Gaussian distributed dataset.",
    params={"contamination": 0.01, "columns": [], "random_state": DEFAULT_RANDOM_STATE},
    learns_from_data=True,
)
class EllipticEnvelopeCalculator(BaseCalculator):
    """Fit one univariate EllipticEnvelope per numeric column at the configured contamination."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # Elliptic envelope filters outlier *rows*; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> EllipticEnvelopeArtifact:  # pylint: disable=arguments-differ
        """Fit the per-column envelopes, skipping columns with fewer than five usable samples."""
        if user_picked_no_columns(config):
            return {}

        contamination = config.get("contamination", 0.01)
        random_state = config.get("random_state", DEFAULT_RANDOM_STATE)
        # TODO(pandas-removal): same per-column coercion/dropna caveat as
        # zscore.py — EllipticEnvelope.fit itself takes numpy fine, but the
        # per-column pd.to_numeric(errors="coerce").dropna() skip-logic for
        # non-numeric/short columns needs a native-Polars equivalent first.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        if not cols:
            return {}

        models: dict[str, Any] = {}
        warnings = []
        for col in cols:
            series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
            if series.shape[0] < 5:
                warnings.append(f"Column '{col}': Too few samples ({series.shape[0]})")
                continue
            try:
                model = EllipticEnvelope(contamination=contamination, random_state=random_state)
                model.fit(series.to_numpy().reshape(-1, 1))
                models[col] = model
            except Exception as e:  # noqa: BLE001 - per-column fit failure is logged and recorded in warnings
                logger.warning(f"EllipticEnvelope fit failed for column {col}: {e}")
                warnings.append(f"Column '{col}': {str(e)}")

        return {
            "type": "elliptic_envelope",
            "models": models,
            "contamination": contamination,
            "warnings": warnings,
        }

fit(X, _y, config)

Fit the per-column envelopes, skipping columns with fewer than five usable samples.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> EllipticEnvelopeArtifact:  # pylint: disable=arguments-differ
    """Fit the per-column envelopes, skipping columns with fewer than five usable samples."""
    if user_picked_no_columns(config):
        return {}

    contamination = config.get("contamination", 0.01)
    random_state = config.get("random_state", DEFAULT_RANDOM_STATE)
    # TODO(pandas-removal): same per-column coercion/dropna caveat as
    # zscore.py — EllipticEnvelope.fit itself takes numpy fine, but the
    # per-column pd.to_numeric(errors="coerce").dropna() skip-logic for
    # non-numeric/short columns needs a native-Polars equivalent first.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    if not cols:
        return {}

    models: dict[str, Any] = {}
    warnings = []
    for col in cols:
        series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
        if series.shape[0] < 5:
            warnings.append(f"Column '{col}': Too few samples ({series.shape[0]})")
            continue
        try:
            model = EllipticEnvelope(contamination=contamination, random_state=random_state)
            model.fit(series.to_numpy().reshape(-1, 1))
            models[col] = model
        except Exception as e:  # noqa: BLE001 - per-column fit failure is logged and recorded in warnings
            logger.warning(f"EllipticEnvelope fit failed for column {col}: {e}")
            warnings.append(f"Column '{col}': {str(e)}")

    return {
        "type": "elliptic_envelope",
        "models": models,
        "contamination": contamination,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py
203
204
205
206
207
208
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # Elliptic envelope filters outlier *rows*; column set is preserved.
    return input_schema

IQRApplier

Bases: BaseApplier

Drop rows outside the fitted per-column IQR bounds; null/NaN values are kept.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
class IQRApplier(BaseApplier):
    """Drop rows outside the fitted per-column IQR bounds; null/NaN values are kept."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned quartile limits without recomputing request statistics."""
        if local_state_fields(
            raw, "iqr", {"type", "bounds", "multiplier", "warnings"}, allow_empty=True
        ):
            validate_fitted_bounds(raw["bounds"], partial=False)
            validate_detector_warnings(raw["warnings"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved row filtering; prediction still rejects any removed rows."""
        if engine not in ("pandas", "polars"):
            return None
        IQRApplier.validate_inference_state(state)
        effect = "filter" if state.get("bounds") else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the fitted per-column bounds on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        mask = pl.repeat(True, pl.len())
        for col, bound in bounds.items():
            if col not in X.columns:
                continue
            values = pl.col(col).cast(pl.Float64, strict=False)
            col_mask = (values >= bound["lower"]) & (values <= bound["upper"])
            missing = values.is_null() | values.is_nan()
            mask = mask & (col_mask | missing)

        mask_series = X.select(mask.alias("mask")).get_column("mask")
        return X.filter(mask_series), _filter_y_polars(y, mask_series)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        mask = pd.Series(True, index=X.index)
        for col, bound in bounds.items():
            if col not in X.columns:
                continue
            series = pd.to_numeric(X[col], errors="coerce")
            col_mask = (series >= bound["lower"]) & (series <= bound["upper"])
            mask = mask & (col_mask | series.isna())

        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the fitted per-column bounds on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
48
49
50
51
52
53
54
55
56
57
58
@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the fitted per-column bounds on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved row filtering; prediction still rejects any removed rows.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
39
40
41
42
43
44
45
46
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved row filtering; prediction still rejects any removed rows."""
    if engine not in ("pandas", "polars"):
        return None
    IQRApplier.validate_inference_state(state)
    effect = "filter" if state.get("bounds") else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "row")

validate_inference_state(raw) staticmethod

Inspect learned quartile limits without recomputing request statistics.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
29
30
31
32
33
34
35
36
37
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned quartile limits without recomputing request statistics."""
    if local_state_fields(
        raw, "iqr", {"type", "bounds", "multiplier", "warnings"}, allow_empty=True
    ):
        validate_fitted_bounds(raw["bounds"], partial=False)
        validate_detector_warnings(raw["warnings"])
    return raw

IQRCalculator

Bases: BaseCalculator

Fit per-column bounds at the quartiles widened by multiplier times the IQR.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
@NodeRegistry.register("IQR", IQRApplier)
@node_meta(
    id="IQR",
    name="IQR Outlier Removal",
    category="Preprocessing",
    description="Remove outliers using Interquartile Range.",
    params={"multiplier": 1.5, "columns": []},
    learns_from_data=True,
)
class IQRCalculator(BaseCalculator):
    """Fit per-column bounds at the quartiles widened by ``multiplier`` times the IQR."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # IQR removes outlier *rows*; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> IQRArtifact:  # pylint: disable=arguments-differ
        """Compute quartile bounds per numeric column; warn on empty or non-numeric ones."""
        if user_picked_no_columns(config):
            return {}

        multiplier = config.get("multiplier", 1.5)
        # TODO(pandas-removal): stays on resolve_columns_then_to_pandas (not
        # ..._to_numpy) because `.quantile()` needs Pandas' linear
        # interpolation semantics, which differ from Polars' default
        # "nearest" interpolation (see _helpers.py note). Switching to numpy
        # would require doing the quantile math with
        # np.percentile(..., method="linear") plus matching Pandas'
        # errors="coerce" NaN-drop behavior before benchmarking equivalence.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        if not cols:
            return {}

        bounds: dict[str, dict[str, float]] = {}
        warnings = []
        for col in cols:
            series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
            if series.empty:
                warnings.append(f"Column '{col}': Empty or non-numeric")
                continue
            q1 = series.quantile(0.25)
            q3 = series.quantile(0.75)
            iqr = q3 - q1
            bounds[col] = {"lower": q1 - (multiplier * iqr), "upper": q3 + (multiplier * iqr)}

        return {
            "type": "iqr",
            "bounds": bounds,
            "multiplier": multiplier,
            "warnings": warnings,
        }

fit(X, _y, config)

Compute quartile bounds per numeric column; warn on empty or non-numeric ones.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> IQRArtifact:  # pylint: disable=arguments-differ
    """Compute quartile bounds per numeric column; warn on empty or non-numeric ones."""
    if user_picked_no_columns(config):
        return {}

    multiplier = config.get("multiplier", 1.5)
    # TODO(pandas-removal): stays on resolve_columns_then_to_pandas (not
    # ..._to_numpy) because `.quantile()` needs Pandas' linear
    # interpolation semantics, which differ from Polars' default
    # "nearest" interpolation (see _helpers.py note). Switching to numpy
    # would require doing the quantile math with
    # np.percentile(..., method="linear") plus matching Pandas'
    # errors="coerce" NaN-drop behavior before benchmarking equivalence.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    if not cols:
        return {}

    bounds: dict[str, dict[str, float]] = {}
    warnings = []
    for col in cols:
        series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
        if series.empty:
            warnings.append(f"Column '{col}': Empty or non-numeric")
            continue
        q1 = series.quantile(0.25)
        q3 = series.quantile(0.75)
        iqr = q3 - q1
        bounds[col] = {"lower": q1 - (multiplier * iqr), "upper": q3 + (multiplier * iqr)}

    return {
        "type": "iqr",
        "bounds": bounds,
        "multiplier": multiplier,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py
107
108
109
110
111
112
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # IQR removes outlier *rows*; column set is preserved.
    return input_schema

IsolationForestApplier

Bases: BaseApplier

Exclude rows using the saved joint forest, retaining unscored nonfinite rows.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
class IsolationForestApplier(BaseApplier):
    """Exclude rows using the saved joint forest, retaining unscored nonfinite rows."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the fitted forest and its feature axes without running tree inference."""
        model = validate_detector(raw, "isolation_forest", IsolationForest)
        if model is None:
            return raw
        trees = getattr(model, "estimators_", None)
        axes = getattr(model, "estimators_features_", None)
        if (
            not isinstance(trees, list)
            or not trees
            or not isinstance(axes, list)
            or len(trees) != len(axes)
        ):
            raise ValueError("Fitted IsolationForest requires matching tree and feature lists.")
        for tree, features in zip(trees, axes, strict=True):
            _validate_tree(tree, features, len(raw["columns"]))
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Declare local row filtering only after validating the fitted detector."""
        if engine not in {"pandas", "polars"}:
            return None
        IsolationForestApplier.validate_inference_state(state)
        return ExecutionCapability(
            engine, "apply", "local", "filter" if state else "preserve", "row"
        )

    @apply_method
    def apply(self, X: Any, y: Any, params: dict) -> Any:
        """Apply one positional mask to features and targets on either native engine."""
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {"pandas": filter_detector_rows, "polars": filter_detector_rows},
        )

apply(X, y, params)

Apply one positional mask to features and targets on either native engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
82
83
84
85
86
87
88
89
@apply_method
def apply(self, X: Any, y: Any, params: dict) -> Any:
    """Apply one positional mask to features and targets on either native engine."""
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {"pandas": filter_detector_rows, "polars": filter_detector_rows},
    )

inference_capability(state, *, engine) staticmethod

Declare local row filtering only after validating the fitted detector.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
72
73
74
75
76
77
78
79
80
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Declare local row filtering only after validating the fitted detector."""
    if engine not in {"pandas", "polars"}:
        return None
    IsolationForestApplier.validate_inference_state(state)
    return ExecutionCapability(
        engine, "apply", "local", "filter" if state else "preserve", "row"
    )

validate_inference_state(raw) staticmethod

Inspect the fitted forest and its feature axes without running tree inference.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the fitted forest and its feature axes without running tree inference."""
    model = validate_detector(raw, "isolation_forest", IsolationForest)
    if model is None:
        return raw
    trees = getattr(model, "estimators_", None)
    axes = getattr(model, "estimators_features_", None)
    if (
        not isinstance(trees, list)
        or not trees
        or not isinstance(axes, list)
        or len(trees) != len(axes)
    ):
        raise ValueError("Fitted IsolationForest requires matching tree and feature lists.")
    for tree, features in zip(trees, axes, strict=True):
        _validate_tree(tree, features, len(raw["columns"]))
    return raw

IsolationForestCalculator

Bases: BaseCalculator

Fit one forest across selected numeric features, excluding the implicit target.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
@NodeRegistry.register("IsolationForest", IsolationForestApplier)
@node_meta(
    id="IsolationForest",
    name="Isolation Forest",
    category="Preprocessing",
    description="Exclude multivariate outliers using a learned joint isolation forest.",
    params={
        "columns": [],
        "n_estimators": 100,
        "max_samples": "auto",
        "contamination": 0.01,
        "random_state": 42,
    },
    learns_from_data=True,
)
class IsolationForestCalculator(BaseCalculator):
    """Fit one forest across selected numeric features, excluding the implicit target."""

    def infer_output_schema(self, input_schema: Any, config: dict) -> Any:
        """Row exclusion preserves input column names and dtypes."""
        return input_schema

    @fit_method
    def fit(self, X: Any, y: Any, config: dict) -> dict:
        """Learn only from finite training rows; malformed options and fit errors propagate."""
        if user_picked_no_columns(config):
            return {}
        return fit_dual_engine(
            (X, y) if y is not None else X, config, {"pandas": _fit, "polars": _fit}
        )

fit(X, y, config)

Learn only from finite training rows; malformed options and fit errors propagate.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
114
115
116
117
118
119
120
121
@fit_method
def fit(self, X: Any, y: Any, config: dict) -> dict:
    """Learn only from finite training rows; malformed options and fit errors propagate."""
    if user_picked_no_columns(config):
        return {}
    return fit_dual_engine(
        (X, y) if y is not None else X, config, {"pandas": _fit, "polars": _fit}
    )

infer_output_schema(input_schema, config)

Row exclusion preserves input column names and dtypes.

Source code in skyulf-core/skyulf/preprocessing/outliers/isolation_forest.py
110
111
112
def infer_output_schema(self, input_schema: Any, config: dict) -> Any:
    """Row exclusion preserves input column names and dtypes."""
    return input_schema

LocalOutlierFactorApplier

Bases: BaseApplier

Exclude unseen rows using the fitted training population as the neighbor reference.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
class LocalOutlierFactorApplier(BaseApplier):
    """Exclude unseen rows using the fitted training population as the neighbor reference."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Require novelty mode and internally consistent learned neighbor statistics."""
        model = validate_detector(raw, "local_outlier_factor", LocalOutlierFactor)
        if model is None:
            return raw
        if model.novelty is not True or model.metric != "minkowski" or model.p != 2:
            raise ValueError("Fitted LocalOutlierFactor requires Euclidean novelty scoring.")
        rows = getattr(model, "n_samples_fit_", 0)
        neighbors = getattr(model, "n_neighbors_", 0)
        if not 1 <= neighbors < rows:
            raise ValueError("Fitted LocalOutlierFactor neighbor count is invalid.")
        fitted_array(model._fit_X, (rows, len(raw["columns"])))
        fitted_array(model._lrd, (rows,), positive=True)
        fitted_array(model.negative_outlier_factor_, (rows,))
        fitted_array(model._distances_fit_X_, (rows, neighbors))
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Declare local row filtering against fixed learned neighbors."""
        if engine not in {"pandas", "polars"}:
            return None
        LocalOutlierFactorApplier.validate_inference_state(state)
        return ExecutionCapability(
            engine, "apply", "local", "filter" if state else "preserve", "row"
        )

    @apply_method
    def apply(self, X: Any, y: Any, params: dict) -> Any:
        """Score new rows only, retaining nonfinite rows for explicit downstream handling."""
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {"pandas": filter_detector_rows, "polars": filter_detector_rows},
        )

apply(X, y, params)

Score new rows only, retaining nonfinite rows for explicit downstream handling.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
73
74
75
76
77
78
79
80
@apply_method
def apply(self, X: Any, y: Any, params: dict) -> Any:
    """Score new rows only, retaining nonfinite rows for explicit downstream handling."""
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {"pandas": filter_detector_rows, "polars": filter_detector_rows},
    )

inference_capability(state, *, engine) staticmethod

Declare local row filtering against fixed learned neighbors.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
63
64
65
66
67
68
69
70
71
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Declare local row filtering against fixed learned neighbors."""
    if engine not in {"pandas", "polars"}:
        return None
    LocalOutlierFactorApplier.validate_inference_state(state)
    return ExecutionCapability(
        engine, "apply", "local", "filter" if state else "preserve", "row"
    )

validate_inference_state(raw) staticmethod

Require novelty mode and internally consistent learned neighbor statistics.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Require novelty mode and internally consistent learned neighbor statistics."""
    model = validate_detector(raw, "local_outlier_factor", LocalOutlierFactor)
    if model is None:
        return raw
    if model.novelty is not True or model.metric != "minkowski" or model.p != 2:
        raise ValueError("Fitted LocalOutlierFactor requires Euclidean novelty scoring.")
    rows = getattr(model, "n_samples_fit_", 0)
    neighbors = getattr(model, "n_neighbors_", 0)
    if not 1 <= neighbors < rows:
        raise ValueError("Fitted LocalOutlierFactor neighbor count is invalid.")
    fitted_array(model._fit_X, (rows, len(raw["columns"])))
    fitted_array(model._lrd, (rows,), positive=True)
    fitted_array(model.negative_outlier_factor_, (rows,))
    fitted_array(model._distances_fit_X_, (rows, neighbors))
    return raw

LocalOutlierFactorCalculator

Bases: BaseCalculator

Learn LOF training scores and novelty state with separate train/apply representations.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
@NodeRegistry.register("LocalOutlierFactor", LocalOutlierFactorApplier)
@node_meta(
    id="LocalOutlierFactor",
    name="Local Outlier Factor",
    category="Preprocessing",
    description="Exclude joint local-density outliers; use fixed neighbors for new rows.",
    params={"columns": [], "n_neighbors": 20, "contamination": 0.01},
    learns_from_data=True,
)
class LocalOutlierFactorCalculator(BaseCalculator):
    """Learn LOF training scores and novelty state with separate train/apply representations."""

    def infer_output_schema(self, input_schema: Any, config: dict) -> Any:
        """Filtering rows retains all input column names and dtypes."""
        return input_schema

    @fit_method
    def fit(self, X: Any, y: Any, config: dict) -> dict:
        """Fit a saved detector; use fit_transform_train to obtain filtered training data."""
        if user_picked_no_columns(config):
            return {}
        return fit_dual_engine(
            (X, y) if y is not None else X, config, {"pandas": _fit, "polars": _fit}
        )

    def fit_transform_train(self, df: Any, config: dict) -> tuple[dict, Any]:
        """Filter training rows with negative_outlier_factor_, never novelty predict."""
        if user_picked_no_columns(config):
            return {}, df
        return fit_transform_train_dual_engine(
            df, config, {"pandas": _fit_train, "polars": _fit_train}
        )

fit(X, y, config)

Fit a saved detector; use fit_transform_train to obtain filtered training data.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
 99
100
101
102
103
104
105
106
@fit_method
def fit(self, X: Any, y: Any, config: dict) -> dict:
    """Fit a saved detector; use fit_transform_train to obtain filtered training data."""
    if user_picked_no_columns(config):
        return {}
    return fit_dual_engine(
        (X, y) if y is not None else X, config, {"pandas": _fit, "polars": _fit}
    )

fit_transform_train(df, config)

Filter training rows with negative_outlier_factor_, never novelty predict.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
108
109
110
111
112
113
114
def fit_transform_train(self, df: Any, config: dict) -> tuple[dict, Any]:
    """Filter training rows with negative_outlier_factor_, never novelty predict."""
    if user_picked_no_columns(config):
        return {}, df
    return fit_transform_train_dual_engine(
        df, config, {"pandas": _fit_train, "polars": _fit_train}
    )

infer_output_schema(input_schema, config)

Filtering rows retains all input column names and dtypes.

Source code in skyulf-core/skyulf/preprocessing/outliers/local_outlier_factor.py
95
96
97
def infer_output_schema(self, input_schema: Any, config: dict) -> Any:
    """Filtering rows retains all input column names and dtypes."""
    return input_schema

ManualBoundsApplier

Bases: BaseApplier

Drop rows outside the user-specified per-column lower/upper bounds.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
class ManualBoundsApplier(BaseApplier):
    """Drop rows outside the user-specified per-column lower/upper bounds."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved optional bounds without applying the row filter."""
        local_state_fields(raw, "manual_bounds", {"type", "bounds"})
        validate_fitted_bounds(raw["bounds"], partial=True)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Declare row-local filtering; prediction rejects requests that lose rows."""
        if engine not in ("pandas", "polars"):
            return None
        ManualBoundsApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "filter", "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the configured bounds on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        # A scalar mask broadcasts for X but yields only index zero when filtering y.
        mask = pl.repeat(True, pl.len())
        for col, bound in bounds.items():
            if col not in X.columns:
                continue
            mask = mask & _manual_bounds_col_mask_polars(col, bound)

        mask_series = X.select(mask.alias("mask")).get_column("mask")
        return X.filter(mask_series), _filter_y_polars(y, mask_series)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        mask = pd.Series(True, index=X.index)
        for col, bound in bounds.items():
            if col not in X.columns:
                continue
            series = pd.to_numeric(X[col], errors="coerce")
            mask = mask & _manual_bounds_col_mask_pandas(series, bound)

        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the configured bounds on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
61
62
63
64
65
66
67
68
69
70
71
@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the configured bounds on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Declare row-local filtering; prediction rejects requests that lose rows.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
53
54
55
56
57
58
59
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Declare row-local filtering; prediction rejects requests that lose rows."""
    if engine not in ("pandas", "polars"):
        return None
    ManualBoundsApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "filter", "row")

validate_inference_state(raw) staticmethod

Inspect saved optional bounds without applying the row filter.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
46
47
48
49
50
51
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved optional bounds without applying the row filter."""
    local_state_fields(raw, "manual_bounds", {"type", "bounds"})
    validate_fitted_bounds(raw["bounds"], partial=True)
    return raw

ManualBoundsCalculator

Bases: BaseCalculator

Package the user-configured bounds into an artifact without reading the data.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
@NodeRegistry.register("ManualBounds", ManualBoundsApplier)
@node_meta(
    id="ManualBounds",
    name="Manual Bounds",
    category="Preprocessing",
    description="Filter outliers by manually specifying lower and upper bounds for columns.",
    params={"bounds": {}},
    learns_from_data=False,
)
class ManualBoundsCalculator(BaseCalculator):
    """Package the user-configured bounds into an artifact without reading the data."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # Manual bounds filter rows; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, _X: Any, _y: Any, config: dict[str, Any]) -> ManualBoundsArtifact:  # pylint: disable=arguments-differ
        """Return the ``bounds`` config as-is; no statistics are computed."""
        # Config: {'bounds': {'col1': {'lower': 0, 'upper': 100}, ...}}
        return {"type": "manual_bounds", "bounds": config.get("bounds", {})}

fit(_X, _y, config)

Return the bounds config as-is; no statistics are computed.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
124
125
126
127
128
@fit_method
def fit(self, _X: Any, _y: Any, config: dict[str, Any]) -> ManualBoundsArtifact:  # pylint: disable=arguments-differ
    """Return the ``bounds`` config as-is; no statistics are computed."""
    # Config: {'bounds': {'col1': {'lower': 0, 'upper': 100}, ...}}
    return {"type": "manual_bounds", "bounds": config.get("bounds", {})}

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/manual_bounds.py
117
118
119
120
121
122
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # Manual bounds filter rows; column set is preserved.
    return input_schema

WinsorizeApplier

Bases: BaseApplier

Clip values to the fitted percentile bounds; rows are never removed.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
class WinsorizeApplier(BaseApplier):
    """Clip values to the fitted percentile bounds; rows are never removed."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned bounds and metadata without recomputing request quantiles."""
        fields = {"type", "bounds", "lower_percentile", "upper_percentile", "warnings"}
        if isinstance(raw, dict):
            fields.update({"percentiles_by_column"}.intersection(raw))
        if not local_state_fields(
            raw,
            "winsorize",
            fields,
            allow_empty=True,
        ):
            return raw
        validate_fitted_bounds(raw["bounds"], partial=False)
        _validate_saved_percentiles(raw)
        for field in ("lower_percentile", "upper_percentile"):
            value = raw[field]
            if isinstance(value, bool) or not isinstance(value, Real) or not 0 <= value <= 100:
                raise ValueError("Fitted percentiles must be numbers between zero and 100.")
        if type(raw["warnings"]) is not list or any(
            type(item) is not str for item in raw["warnings"]
        ):
            raise ValueError("Fitted warnings must be a list of strings.")
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local row clipping with saved limits without admitting Spark workers."""
        if engine not in ("pandas", "polars"):
            return None
        WinsorizeApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Clip ``X`` to the fitted per-column bounds; ``y`` passes through untouched."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y. Winsorize never
        # filters rows, but the wrap keeps behavior consistent with the other
        # outlier nodes and avoids losing y from the returned tuple.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        numeric_cols = set(auto_detect_numeric_columns(X))
        exprs = []
        for col, bound in bounds.items():
            if col not in X.columns or col not in numeric_cols:
                continue
            exprs.append(_clip_polars_column(X, col, bound))
        return X.with_columns(exprs), y

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        df_out = X.copy()
        for col, bound in bounds.items():
            if col not in df_out.columns:
                continue
            if pd.api.types.is_numeric_dtype(df_out[col]) or is_decimal_series(df_out[col]):
                df_out[col] = _clip_pandas_column(df_out[col], bound, col)
        return df_out, y

apply(X, y, params)

Clip X to the fitted per-column bounds; y passes through untouched.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
182
183
184
185
186
187
188
189
190
191
192
@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Clip ``X`` to the fitted per-column bounds; ``y`` passes through untouched."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y. Winsorize never
    # filters rows, but the wrap keeps behavior consistent with the other
    # outlier nodes and avoids losing y from the returned tuple.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe local row clipping with saved limits without admitting Spark workers.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
174
175
176
177
178
179
180
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local row clipping with saved limits without admitting Spark workers."""
    if engine not in ("pandas", "polars"):
        return None
    WinsorizeApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect learned bounds and metadata without recomputing request quantiles.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned bounds and metadata without recomputing request quantiles."""
    fields = {"type", "bounds", "lower_percentile", "upper_percentile", "warnings"}
    if isinstance(raw, dict):
        fields.update({"percentiles_by_column"}.intersection(raw))
    if not local_state_fields(
        raw,
        "winsorize",
        fields,
        allow_empty=True,
    ):
        return raw
    validate_fitted_bounds(raw["bounds"], partial=False)
    _validate_saved_percentiles(raw)
    for field in ("lower_percentile", "upper_percentile"):
        value = raw[field]
        if isinstance(value, bool) or not isinstance(value, Real) or not 0 <= value <= 100:
            raise ValueError("Fitted percentiles must be numbers between zero and 100.")
    if type(raw["warnings"]) is not list or any(
        type(item) is not str for item in raw["warnings"]
    ):
        raise ValueError("Fitted warnings must be a list of strings.")
    return raw

WinsorizeCalculator

Bases: BaseCalculator

Fit per-column clip bounds at the configured lower/upper percentiles.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
@NodeRegistry.register("Winsorize", WinsorizeApplier)
@node_meta(
    id="Winsorize",
    name="Winsorization",
    category="Preprocessing",
    description="Limit extreme values in the data.",
    params={
        "lower_percentile": 5.0,
        "upper_percentile": 95.0,
        "columns": [],
        "percentiles_by_column": {},
    },
    learns_from_data=True,
)
class WinsorizeCalculator(BaseCalculator):
    """Fit per-column clip bounds at the configured lower/upper percentiles."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Preserve integer columns and retain existing promotion for other selected types."""
        if not isinstance(input_schema, SkyulfSchema):
            return input_schema
        output = promote_configured_columns_to_float64(input_schema, config)
        for column, dtype in input_schema.dtypes.items():
            if dtype.lower().startswith(("int", "uint")):
                output = output.with_dtype(column, dtype)
        return output

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> WinsorizeArtifact:  # pylint: disable=arguments-differ
        """Compute each column's lower/upper quantiles; warn on empty or non-numeric ones."""
        if user_picked_no_columns(config):
            return {}

        lower_p = config.get("lower_percentile", 5.0)
        upper_p = config.get("upper_percentile", 95.0)
        X = X.to_native() if hasattr(X, "to_native") else X
        cols = resolve_columns(X, config, detect_numeric_columns)
        if not cols:
            return {}
        percentiles = _column_percentiles(config, cols)

        bounds: dict[str, dict[str, float]] = {}
        warnings = []
        for col in cols:
            series, integer = _fit_series(X, col)
            if series.empty:
                warnings.append(f"Column '{col}': Empty or non-numeric")
                continue
            bounds[col] = {
                "lower": _fit_bound(series, percentiles[col]["lower"], col, integer=integer),
                "upper": _fit_bound(series, percentiles[col]["upper"], col, integer=integer),
            }
            if integer:
                bounds[col] = _pandas_integer_bounds(series, bounds[col], col)

        artifact: WinsorizeArtifact = {
            "type": "winsorize",
            "bounds": bounds,
            "lower_percentile": lower_p,
            "upper_percentile": upper_p,
            "warnings": warnings,
        }
        if "percentiles_by_column" in config:
            artifact["percentiles_by_column"] = {col: percentiles[col] for col in bounds}
        return artifact

fit(X, _y, config)

Compute each column's lower/upper quantiles; warn on empty or non-numeric ones.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> WinsorizeArtifact:  # pylint: disable=arguments-differ
    """Compute each column's lower/upper quantiles; warn on empty or non-numeric ones."""
    if user_picked_no_columns(config):
        return {}

    lower_p = config.get("lower_percentile", 5.0)
    upper_p = config.get("upper_percentile", 95.0)
    X = X.to_native() if hasattr(X, "to_native") else X
    cols = resolve_columns(X, config, detect_numeric_columns)
    if not cols:
        return {}
    percentiles = _column_percentiles(config, cols)

    bounds: dict[str, dict[str, float]] = {}
    warnings = []
    for col in cols:
        series, integer = _fit_series(X, col)
        if series.empty:
            warnings.append(f"Column '{col}': Empty or non-numeric")
            continue
        bounds[col] = {
            "lower": _fit_bound(series, percentiles[col]["lower"], col, integer=integer),
            "upper": _fit_bound(series, percentiles[col]["upper"], col, integer=integer),
        }
        if integer:
            bounds[col] = _pandas_integer_bounds(series, bounds[col], col)

    artifact: WinsorizeArtifact = {
        "type": "winsorize",
        "bounds": bounds,
        "lower_percentile": lower_p,
        "upper_percentile": upper_p,
        "warnings": warnings,
    }
    if "percentiles_by_column" in config:
        artifact["percentiles_by_column"] = {col: percentiles[col] for col in bounds}
    return artifact

infer_output_schema(input_schema, config)

Preserve integer columns and retain existing promotion for other selected types.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py
240
241
242
243
244
245
246
247
248
249
250
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Preserve integer columns and retain existing promotion for other selected types."""
    if not isinstance(input_schema, SkyulfSchema):
        return input_schema
    output = promote_configured_columns_to_float64(input_schema, config)
    for column, dtype in input_schema.dtypes.items():
        if dtype.lower().startswith(("int", "uint")):
            output = output.with_dtype(column, dtype)
    return output

ZScoreApplier

Bases: BaseApplier

Drop rows whose |z| against the fitted mean/std exceeds threshold in any column.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
class ZScoreApplier(BaseApplier):
    """Drop rows whose |z| against the fitted mean/std exceeds ``threshold`` in any column."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved numeric statistics and native scalar threshold semantics."""
        if local_state_fields(
            raw, "zscore", {"type", "stats", "threshold", "warnings"}, allow_empty=True
        ):
            _validate_zscore_statistics(raw["stats"])
            if not isinstance(raw["threshold"], (Real, Decimal)):
                local_scalar(raw["threshold"], "ZScore threshold")
            validate_detector_warnings(raw["warnings"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved row filtering without bypassing the prediction row-count guard."""
        if engine not in ("pandas", "polars"):
            return None
        ZScoreApplier.validate_inference_state(state)
        effect = "filter" if state.get("stats") else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the fitted z-score threshold on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        stats = params.get("stats", {})
        threshold = params.get("threshold", 3.0)
        if not stats:
            return X, y

        mask = pl.repeat(True, pl.len())
        for col, stat in stats.items():
            if col not in X.columns or stat["std"] == 0:
                continue
            values = pl.col(col).cast(pl.Float64, strict=False)
            z = (values - stat["mean"]) / stat["std"]
            col_mask = z.abs() <= threshold
            missing = values.is_null() | values.is_nan()
            mask = mask & (col_mask | missing)

        mask_series = X.select(mask.alias("mask")).get_column("mask")
        return X.filter(mask_series), _filter_y_polars(y, mask_series)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        stats = params.get("stats", {})
        threshold = params.get("threshold", 3.0)
        if not stats:
            return X, y

        mask = pd.Series(True, index=X.index)
        for col, stat in stats.items():
            if col not in X.columns or stat["std"] == 0:
                continue
            series = pd.to_numeric(X[col], errors="coerce")
            z = (series - stat["mean"]) / stat["std"]
            col_mask = z.abs() <= threshold
            mask = mask & (col_mask | series.isna())

        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the fitted z-score threshold on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
61
62
63
64
65
66
67
68
69
70
71
@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the fitted z-score threshold on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved row filtering without bypassing the prediction row-count guard.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
52
53
54
55
56
57
58
59
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved row filtering without bypassing the prediction row-count guard."""
    if engine not in ("pandas", "polars"):
        return None
    ZScoreApplier.validate_inference_state(state)
    effect = "filter" if state.get("stats") else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "row")

validate_inference_state(raw) staticmethod

Inspect saved numeric statistics and native scalar threshold semantics.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
40
41
42
43
44
45
46
47
48
49
50
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved numeric statistics and native scalar threshold semantics."""
    if local_state_fields(
        raw, "zscore", {"type", "stats", "threshold", "warnings"}, allow_empty=True
    ):
        _validate_zscore_statistics(raw["stats"])
        if not isinstance(raw["threshold"], (Real, Decimal)):
            local_scalar(raw["threshold"], "ZScore threshold")
        validate_detector_warnings(raw["warnings"])
    return raw

ZScoreCalculator

Bases: BaseCalculator

Fit per-column mean and population std (ddof=0) for z-score filtering.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
@NodeRegistry.register("ZScore", ZScoreApplier)
@node_meta(
    id="ZScore",
    name="Z-Score Outlier Removal",
    category="Preprocessing",
    description="Remove outliers using Z-Score.",
    params={"threshold": 3.0, "columns": []},
    learns_from_data=True,
)
class ZScoreCalculator(BaseCalculator):
    """Fit per-column mean and population std (``ddof=0``) for z-score filtering."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # Z-score removes outlier *rows*; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ZScoreArtifact:  # pylint: disable=arguments-differ
        """Record each column's mean/std; warn on empty or zero-variance columns."""
        if user_picked_no_columns(config):
            return {}

        threshold = config.get("threshold", 3.0)
        # TODO(pandas-removal): mean/std alone would be numpy-safe (Polars std
        # ddof=0 already matches Pandas — verified), but this loop relies on
        # per-column pd.to_numeric(errors="coerce") to gracefully skip
        # non-numeric/mixed-dtype columns and .dropna() per column before
        # computing stats. Revisit once there's a native Polars equivalent
        # (pl.col(col).cast(pl.Float64, strict=False).drop_nulls()) wired in
        # to replace the coercion step without changing which columns get a
        # "non-numeric" warning.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        if not cols:
            return {}

        stats: dict[str, dict[str, float]] = {}
        warnings = []
        for col in cols:
            series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
            series = series[np.isfinite(series)]
            if series.empty:
                warnings.append(f"Column '{col}': Empty, non-numeric or non-finite")
                continue
            std = series.std(ddof=0)
            if std == 0:
                warnings.append(f"Column '{col}': Zero variance (std=0)")
                continue
            stats[col] = {"mean": series.mean(), "std": std}

        return {
            "type": "zscore",
            "stats": stats,
            "threshold": threshold,
            "warnings": warnings,
        }

fit(X, _y, config)

Record each column's mean/std; warn on empty or zero-variance columns.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ZScoreArtifact:  # pylint: disable=arguments-differ
    """Record each column's mean/std; warn on empty or zero-variance columns."""
    if user_picked_no_columns(config):
        return {}

    threshold = config.get("threshold", 3.0)
    # TODO(pandas-removal): mean/std alone would be numpy-safe (Polars std
    # ddof=0 already matches Pandas — verified), but this loop relies on
    # per-column pd.to_numeric(errors="coerce") to gracefully skip
    # non-numeric/mixed-dtype columns and .dropna() per column before
    # computing stats. Revisit once there's a native Polars equivalent
    # (pl.col(col).cast(pl.Float64, strict=False).drop_nulls()) wired in
    # to replace the coercion step without changing which columns get a
    # "non-numeric" warning.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    if not cols:
        return {}

    stats: dict[str, dict[str, float]] = {}
    warnings = []
    for col in cols:
        series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
        series = series[np.isfinite(series)]
        if series.empty:
            warnings.append(f"Column '{col}': Empty, non-numeric or non-finite")
            continue
        std = series.std(ddof=0)
        if std == 0:
            warnings.append(f"Column '{col}': Zero variance (std=0)")
            continue
        stats[col] = {"mean": series.mean(), "std": std}

    return {
        "type": "zscore",
        "stats": stats,
        "threshold": threshold,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py
124
125
126
127
128
129
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # Z-score removes outlier *rows*; column set is preserved.
    return input_schema