Data Validation and Sanitization Best Practices
In this tutorial, you'll learn about Data Validation and Sanitization Best Practices. We cover key concepts, practical examples, and best practices to help you understand and apply this topic effectively.
Data validation is the Process of verifying that input data conforms to expected formats, types, and constraints before it reaches business logic, preventing corrupt data and security vulnerabilities.
What You'll Learn
By the end of this tutorial, you will implement schema validation with Pydantic (Python), Zod (TypeScript), and Joi (Node.js), sanitize inputs against injection attacks, enforce business rules, and build a multi-layer validation pipeline.
Why It Matters
Invalid data corrupts databases, causes application crashes, and enables injection attacks. The OWASP Top 10 lists injection as the number one web security risk. Proper validation and sanitization prevent SQL Injection, XSS, Command Injection, and mass assignment vulnerabilities.
Real-World Use
Durga Antivirus Pro validates threat intelligence feeds against a strict schema before ingesting them into the detection database. Malformed or malicious payloads are rejected at the API boundary before reaching any processing logic.
Validation Flow Architecture
graph TD
A[HTTP Request] --> B[Transport Layer - HTTPS only]
B --> C[Middleware - Body Parser]
C --> D[Schema Validation]
D --> E[Sanitization Layer]
E --> F[Business Logic]
F --> G[Output Encoding]
G --> H[Response]
D -- Reject --> I[400 Bad Request]
E -- Sanitize --> F
style D fill:#f90,color:#fff
style E fill:#4CAF50,color:#fff
Validation occurs at multiple layers: transport, schema, sanitization, and output encoding. Each layer catches a different class of errors.
Python: Pydantic Validation
# validators.py
# Data validation with Pydantic
from pydantic import BaseModel, Field, EmailStr, field_validator, model_validator
from datetime import datetime
from typing import Optional, List
from enum import Enum
class UserRole(str, Enum):
admin = "admin"
editor = "editor"
viewer = "viewer"
class Address(BaseModel):
street: str = Field(..., min_length=5, max_length=200)
city: str = Field(..., min_length=2, max_length=100)
zip_code: str = Field(..., pattern=r'^\d{5}(-\d{4})?$')
country: str = Field(..., min_length=2, max_length=2)
class UserCreate(BaseModel):
"""User creation request validation schema."""
username: str = Field(
..., min_length=3, max_length=50,
pattern=r'^[a-zA-Z0-9_]+$',
description="Alphanumeric username (underscores allowed)"
)
email: EmailStr
password: str = Field(..., min_length=8, max_length=128)
age: int = Field(..., ge=13, le=120)
role: UserRole = UserRole.viewer
address: Optional[Address] = None
tags: List[str] = Field(default_factory=list, max_length=10)
@field_validator('password')
@classmethod
def password_strength(cls, v):
"""Enforce password complexity rules."""
if not any(c.isupper() for c in v):
raise ValueError('Password must contain an uppercase letter')
if not any(c.isdigit() for c in v):
raise ValueError('Password must contain a digit')
if not any(c in '!@#$%^&*' for c in v):
raise ValueError('Password must contain a special character')
return v
@field_validator('username')
@classmethod
def no_reserved_names(cls, v):
"""Block reserved usernames."""
reserved = {'admin', 'root', 'system', 'dodatech'}
if v.lower() in reserved:
raise ValueError('This username is reserved')
return v
@model_validator(mode='after')
def check_email_not_username(self):
"""Prevent username/email reuse pattern."""
if self.username.lower() in self.email.lower():
raise ValueError('Username cannot be part of email')
# Usage
try:
user = UserCreate(
username='alice_dev',
email='alice@example.com',
password='SecurePass1!',
age=28,
role='editor',
)
print(f"Validated: {user.model_dump()}")
except Exception as e:
print(f"Validation error: {e}")
Expected output:
Validated: {'username': 'alice_dev', 'email': 'alice@example.com', 'password': '********', 'age': 28, 'role': <UserRole.editor: 'editor'>, 'address': None, 'tags': []}
TypeScript: Zod Schema
// validation.ts
// runtime validation with Zod
import { z } from 'zod';
const AddressSchema = z.object({
street: z.string().min(5).max(200),
city: z.string().min(2).max(100),
zipCode: z.string().regex(/^\d{5}(-\d{4})?$/),
country: z.string().length(2),
});
const UserCreateSchema = z.object({
username: z
.string()
.min(3)
.max(50)
.regex(/^[a-zA-Z0-9_]+$/, 'Alphanumeric username required')
.refine((val) => !['admin', 'root', 'system', 'dodatech'].includes(val.toLowerCase()), {
message: 'Username is reserved',
}),
email: z.string().email(),
password: z
.string()
.min(8)
.max(128)
.refine((val) => /[A-Z]/.test(val), 'Uppercase letter required')
.refine((val) => /[0-9]/.test(val), 'Digit required')
.refine((val) => /[!@#$%^&*]/.test(val), 'Special character required'),
age: z.number().int().min(13).max(120),
role: z.enum(['admin', 'editor', 'viewer']).default('viewer'),
address: AddressSchema.optional(),
tags: z.array(z.string().max(50)).max(10).default([]),
});
type UserCreate = z.infer<typeof UserCreateSchema>;
function validateUser(data: unknown): UserCreate {
const result = UserCreateSchema.safeParse(data);
if (!result.success) {
const errors = result.error.issues.map(
(issue) => `${issue.path.join('.')}: ${issue.message}`
);
throw new Error(`Validation failed: ${errors.join(', ')}`);
}
return result.data;
}
Input Sanitization
# sanitizer.py
# Input sanitization against injection attacks
import HTML
import re
import bleach
from urllib.parse import urlparse
class Sanitizer:
"""Multi-purpose input sanitizer for preventing injection."""
ALLOWED_TAGS = ['p', 'br', 'strong', 'em', 'a', 'ul', 'ol', 'li']
ALLOWED_ATTRS = {'a': ['href', 'rel']}
@staticmethod
def sanitize_HTML(value: str) -> str:
"""Remove dangerous HTML tags and attributes."""
if not value:
return value
return bleach.clean(
value,
tags=Sanitizer.ALLOWED_TAGS,
attributes=Sanitizer.ALLOWED_ATTRS,
strip=True
)
@staticmethod
def sanitize_string(value: str, max_length: int = 1000) -> str:
"""Strip dangerous characters from a string."""
if not value:
return value
# Remove control characters except newlines
cleaned = re.sub(R'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]', '', value)
# Limit length
return cleaned[:max_length]
@staticmethod
def sanitize_URL(value: str) -> str:
"""Allow only safe URL schemes."""
if not value:
return ''
parsed = urlparse(value)
allowed_schemes = {'HTTPS', 'HTTP', 'mailto'}
if parsed.scheme not in allowed_schemes:
return ''
# Block JavaScript: and data: URLs
dangerous_patterns = [
R'JavaScript:', R'data:', R'vbscript:',
R'<script', R'onclick', R'onerror',
]
for pattern in dangerous_patterns:
if re.search(pattern, value, re.IGNORECASE):
return ''
return value[:2048]
@staticmethod
def sanitize_filename(value: str) -> str:
"""Remove path traversal and dangerous characters."""
# Remove path separators
cleaned = re.sub(R'[\\/]', '', value)
# Remove null bytes
cleaned = cleaned.replace('\x00', '')
# Allow only safe characters
cleaned = re.sub(R'[^\w\-\. ]', '', cleaned)
return cleaned[:255]
# Usage
sanitizer = Sanitizer()
user_input = '<script>alert("XSS")</script><p>Hello</p>'
print(sanitizer.sanitize_HTML(user_input))
# Output: <p>Hello</p>
SQL Injection Prevention
# SQL_safe.py
# Parameterized queries prevent SQL Injection
import sqlite3
class UserRepository:
"""Safe database access using parameterized queries."""
def __init__(self, db_path: str):
self.db_path = db_path
def get_user_by_id(self, user_id: int):
"""SAFE: Parameterized query prevents injection."""
conn = sqlite3.connect(self.db_path)
Cursor = conn.Cursor()
Cursor.execute(
"SELECT id, username, email FROM users WHERE id = ?",
(user_id,)
)
return Cursor.fetchone()
def create_user(self, username: str, email: str):
"""SAFE: All user input goes through parameters."""
conn = sqlite3.connect(self.db_path)
Cursor = conn.Cursor()
Cursor.execute(
"INSERT INTO users (username, email) VALUES (?, ?)",
(username, email)
)
conn.commit()
return Cursor.lastrowid
# DANGEROUS: Never do this
def unsafe_get_user(user_id: str, Cursor):
"""DANGEROUS: String formatting allows SQL Injection."""
Cursor.execute(f"SELECT * FROM users WHERE id = {user_id}")
Table: Validation Strategies
| Technique | Prevents | Implementation |
|---|---|---|
| Schema validation | Wrong types, missing fields | Pydantic, Zod, Joi |
| Type coercion | Type confusion attacks | Python's int(), TypeScript's z.coerce |
| Length limits | Buffer overflow, storage abuse | max_length, min_length |
| Regex patterns | Format violations | pattern, matches |
| Whitelisting | Unexpected values | Enum, allowlist sets |
| Parameterized queries | SQL Injection | ? placeholders, ORM |
| Output encoding | XSS | Bleach, html.escape |
| Rate Limiting | Brute force | Token bucket, Redis |
Common Errors
1. Client-Side Only Validation
Client-side validation is for user experience only. Every request must be re-validated server-side because client validation is trivially bypassed with tools like curl or Postman.
2. Trusting Deserialized Data
Parsing JSON, YAML, or XML from untrusted sources without schema validation can introduce unexpected fields (mass assignment). Always validate with a schema after deserialization and strip unknown fields.
3. Insufficient Numeric Validation
Accepting negative numbers for positive-only fields (age, quantity) or floating-point where integers are required causes subtle bugs. Use ge (greater or equal) and int type constraints explicitly.
4. Blacklisting Instead of Whitelisting
Blacklisting dangerous patterns (SQL keywords, XSS payloads) always misses edge cases. Whitelist allowed values, formats, and characters to define what IS valid rather than what IS NOT.
5. Double Encoding Issues
URL-encoded input passed through multiple decoding stages can bypass sanitization. Decode and validate input once at the boundary, then work with the decoded value internally.
6. Not Validating File Uploads
Uploaded files must be validated for MIME type (magic bytes), size limits, filename safety, and content scanning. An attacker can upload a PHP webshell named image.jpg if only extension is checked.
Practice Questions
1. What is the difference between validation and sanitization?
Validation checks if input conforms to rules (type, format, range) and rejects non-conforming input. Sanitization modifies input by removing or transforming dangerous content (HTML tags, control characters) while preserving valid content.
2. Why use schema validation libraries instead of manual if-statements?
Schema libraries (Pydantic, Zod, Joi) provide declarative rules, automatic type coercion, detailed error messages, and nested object validation. Manual if-statements are prone to missing edge cases and are harder to maintain.
3. How does parameterized query prevent SQL Injection?
Parameterized queries separate SQL code from data values. The database treats parameters as data, never executable code. Even if a user passes '; DROP TABLE users; -- as a value, the database safely escapes it as a string literal.
4. What is the principle of Least Privilege in validation?
Only accept the minimum data required. Use DTOs (Data Transfer Objects) that expose exactly the fields the client should submit. Never accept raw domain objects or database entities from client input.
5. Challenge: Build a multi-layer validation pipeline for a REST API that validates incoming JSON against a Pydantic schema, sanitizes string fields for HTML and XSS, rejects requests exceeding 10KB payload size, logs all validation failures for security monitoring, and returns RFC 7807 Problem Details for validation errors. Include a rate limiter that blocks clients with more than 5 validation failures per minute.
Mini Project: Express Validation Middleware
// validation-middleware.js
// Reusable validation middleware with Zod
const { z } = require('zod');
const schemas = {
createUser: z.object({
username: z.string().min(3).max(50).regex(/^[a-zA-Z0-9_]+$/),
email: z.string().email(),
password: z.string().min(8).max(128),
age: z.number().int().min(13).max(120).optional(),
}),
updateProfile: z.object({
displayName: z.string().min(1).max(100),
bio: z.string().max(500).optional(),
website: z.string().URL().optional(),
}),
createPost: z.object({
title: z.string().min(3).max(200),
content: z.string().min(1).max(50000),
tags: z.array(z.string().max(30)).max(10).optional(),
published: z.boolean().default(false),
}),
};
function validate(schemaName) {
return (req, res, next) => {
const schema = schemas[schemaName];
if (!schema) {
return res.status(500).JSON({ error: 'Unknown validation schema' });
}
const result = schema.safeParse(req.body);
if (!result.success) {
const errors = result.error.issues.map((issue) => ({
field: issue.path.join('.'),
message: issue.message,
code: issue.code,
}));
return res.status(422).JSON({
type: 'HTTPS://httpwg.org/specs/rfc7807.HTML',
title: 'Validation Error',
status: 422,
errors,
});
}
// Replace body with validated and coerced data
req.body = result.data;
next();
};
}
// Usage in Express routes:
// const { validate } = require('./validation-middleware');
// router.post('/users', validate('createUser'), userController.create);
FAQ
Related Concepts
What's Next
You now understand data validation and sanitization. Next, learn about request validation pipelines as middleware, then explore backend security for comprehensive protection.
- Practice daily — Add Pydantic/Zod validation to an existing API endpoint
- Build a project — Build a validation middleware library that handles schema validation, sanitization, error formatting, and logging
- Explore related topics — Check out OWASP input validation cheat sheet and JSON Schema specification
Built by the developers of Doda Browser, DodaZIP, and Durga Antivirus Pro.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro