{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation & Quality with PyCharter\n",
    "\n",
    "Validate data against schemas/contracts and run quality assessments: basic and batch validation, contract-based validation, quality checks, thresholds, and data profiling.\n",
    "\n",
    "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/optophi/pycharter/blob/main/docs/notebooks/04_validation_quality.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Basic Validation\n",
    "\n",
    "Generate a Pydantic model from JSON Schema and validate records."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycharter import from_dict, validate\n",
    "\n",
    "schema = {\n",
    "    \"type\": \"object\",\n",
    "    \"version\": \"1.0.0\",\n",
    "    \"properties\": {\n",
    "        \"name\": {\"type\": \"string\", \"minLength\": 1},\n",
    "        \"email\": {\"type\": \"string\", \"format\": \"email\"},\n",
    "        \"age\": {\"type\": \"integer\", \"minimum\": 0},\n",
    "    },\n",
    "    \"required\": [\"name\", \"email\"],\n",
    "}\n",
    "\n",
    "User = from_dict(schema, \"User\")\n",
    "\n",
    "result = validate(User, {\"name\": \"Alice\", \"email\": \"alice@test.com\", \"age\": 30})\n",
    "print(\"Valid:\", result.is_valid, \"-\", result.data.name if result.is_valid else result.errors)\n",
    "\n",
    "result = validate(User, {\"name\": \"\", \"email\": \"invalid\", \"age\": -5})\n",
    "print(\"Invalid:\", result.is_valid, \"-\", len(result.errors), \"errors\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Batch Validation\n",
    "\n",
    "Validate multiple records at once with the same model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycharter import validate_batch\n",
    "\n",
    "Product = from_dict(\n",
    "    {\n",
    "        \"type\": \"object\",\n",
    "        \"version\": \"1.0.0\",\n",
    "        \"properties\": {\"id\": {\"type\": \"string\"}, \"price\": {\"type\": \"number\", \"minimum\": 0}},\n",
    "        \"required\": [\"id\", \"price\"],\n",
    "    },\n",
    "    \"Product\",\n",
    ")\n",
    "\n",
    "records = [\n",
    "    {\"id\": \"p1\", \"price\": 10.99},\n",
    "    {\"id\": \"p2\", \"price\": 20.50},\n",
    "    {\"id\": \"p3\", \"price\": -5.0},\n",
    "    {\"id\": \"p4\"},\n",
    "]\n",
    "results = validate_batch(Product, records)\n",
    "valid = sum(1 for r in results if r.is_valid)\n",
    "print(f\"Valid: {valid}/{len(records)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Contract-Based Validation\n",
    "\n",
    "Validate with a contract dict (no model generation); coercion is applied (e.g. string → float)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycharter import validate_with_contract, validate_batch_with_contract\n",
    "\n",
    "contract = {\n",
    "    \"schema\": {\n",
    "        \"type\": \"object\",\n",
    "        \"version\": \"1.0.0\",\n",
    "        \"properties\": {\n",
    "            \"order_id\": {\"type\": \"string\"},\n",
    "            \"total\": {\"type\": \"number\", \"minimum\": 0},\n",
    "            \"status\": {\"type\": \"string\", \"enum\": [\"pending\", \"shipped\", \"delivered\"]},\n",
    "        },\n",
    "        \"required\": [\"order_id\", \"total\", \"status\"],\n",
    "    },\n",
    "    \"coercion_rules\": {\"rules\": {\"total\": \"coerce_to_float\"}},\n",
    "}\n",
    "\n",
    "result = validate_with_contract(\n",
    "    contract, {\"order_id\": \"ord-123\", \"total\": \"99.99\", \"status\": \"pending\"}\n",
    ")\n",
    "print(\"Single:\", result.is_valid, \"total type:\", type(result.data.total).__name__ if result.is_valid else \"N/A\")\n",
    "\n",
    "records = [\n",
    "    {\"order_id\": \"o1\", \"total\": 10.0, \"status\": \"pending\"},\n",
    "    {\"order_id\": \"o2\", \"total\": 20.0, \"status\": \"shipped\"},\n",
    "]\n",
    "results = validate_batch_with_contract(contract, records)\n",
    "print(\"Batch:\", sum(r.is_valid for r in results), \"/\", len(records), \"valid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Quality Check\n",
    "\n",
    "Run a full quality assessment: metrics, violations, and optional thresholds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycharter import QualityCheck, QualityCheckOptions, QualityThresholds\n",
    "\n",
    "contract = {\n",
    "    \"schema\": {\n",
    "        \"type\": \"object\",\n",
    "        \"version\": \"1.0.0\",\n",
    "        \"properties\": {\n",
    "            \"user_id\": {\"type\": \"string\"},\n",
    "            \"name\": {\"type\": \"string\", \"minLength\": 1},\n",
    "            \"email\": {\"type\": \"string\", \"format\": \"email\"},\n",
    "        },\n",
    "        \"required\": [\"user_id\", \"name\", \"email\"],\n",
    "    }\n",
    "}\n",
    "data = [\n",
    "    {\"user_id\": \"1\", \"name\": \"Alice\", \"email\": \"alice@test.com\"},\n",
    "    {\"user_id\": \"2\", \"name\": \"Bob\", \"email\": \"invalid\"},\n",
    "    {\"user_id\": \"3\", \"name\": \"\", \"email\": \"charlie@test.com\"},\n",
    "]\n",
    "\n",
    "check = QualityCheck()\n",
    "report = check.run(\n",
    "    contract=contract,\n",
    "    data=data,\n",
    "    options=QualityCheckOptions(\n",
    "        calculate_metrics=True,\n",
    "        record_violations=True,\n",
    "        include_field_metrics=True,\n",
    "    ),\n",
    ")\n",
    "\n",
    "print(\"Records:\", report.record_count)\n",
    "print(\"Valid:\", report.valid_count)\n",
    "if report.quality_score:\n",
    "    print(\"Score:\", report.quality_score.overall_score, \"/ 100\")\n",
    "    print(\"Accuracy:\", f\"{report.quality_score.accuracy:.1%}\")\n",
    "    print(\"Completeness:\", f\"{report.quality_score.completeness:.1%}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Quality Thresholds\n",
    "\n",
    "Check if data meets minimum score and maximum violation rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresholds = QualityThresholds(\n",
    "    min_overall_score=90.0,\n",
    "    max_violation_rate=0.1,\n",
    "    min_accuracy=0.9,\n",
    ")\n",
    "\n",
    "contract = {\n",
    "    \"schema\": {\n",
    "        \"type\": \"object\",\n",
    "        \"version\": \"1.0.0\",\n",
    "        \"properties\": {\"id\": {\"type\": \"string\"}, \"value\": {\"type\": \"number\", \"minimum\": 0}},\n",
    "        \"required\": [\"id\", \"value\"],\n",
    "    }\n",
    "}\n",
    "data = [{\"id\": \"1\", \"value\": 10}, {\"id\": \"2\", \"value\": 20}, {\"id\": \"3\", \"value\": -5}]\n",
    "\n",
    "report = QualityCheck().run(\n",
    "    contract=contract,\n",
    "    data=data,\n",
    "    options=QualityCheckOptions(\n",
    "        calculate_metrics=True,\n",
    "        check_thresholds=True,\n",
    "        thresholds=thresholds,\n",
    "    ),\n",
    ")\n",
    "print(\"Score:\", report.quality_score.overall_score if report.quality_score else \"N/A\")\n",
    "print(\"Passed:\", report.passed)\n",
    "if report.threshold_breaches:\n",
    "    print(\"Breaches:\", report.threshold_breaches)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Data Profiling\n",
    "\n",
    "Profile data without schema validation (completeness, types, nulls)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycharter import DataProfiler\n",
    "\n",
    "data = [\n",
    "    {\"name\": \"Alice\", \"age\": 30, \"score\": 95.5},\n",
    "    {\"name\": \"Bob\", \"age\": 25, \"score\": 87.2},\n",
    "    {\"name\": \"Charlie\", \"age\": None, \"score\": 92.0},\n",
    "]\n",
    "profiler = DataProfiler()\n",
    "profile = profiler.profile(data)\n",
    "print(\"Record count:\", profile[\"record_count\"])\n",
    "print(\"Average completeness:\", f\"{profile['overall_stats']['average_completeness']:.1%}\")\n",
    "for field, stats in profile[\"field_profiles\"].items():\n",
    "    null_pct = stats[\"null_count\"] / profile[\"record_count\"] * 100\n",
    "    print(f\"  {field}: type={stats['type']}, nulls={null_pct:.0f}%\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}