data-engineering
type: Ontology
identifier: data-engineering
version: 0.4.0
description: Data engineering domain ontology
has namespace:
  semantic memory: semantic
  procedural memory: procedural
defines entity type: data-contract
defines entity type: data-product
defines entity type: data-governance-policy
defines entity type: data-quality-rule
defines entity type: storage-architecture
defines entity type: pipeline-pattern
defines entity type: data-platform
defines trait:
  cited: cited
defines relationship:
  enforces: enforces
  governs: governs
  implements: implements
  governed by: governed_by
discovery config:
  discovery enabled: true
  confidence threshold: 0.8
  has discovery pattern: https://mif-spec.dev/ns/pattern/0
  has discovery pattern: https://mif-spec.dev/ns/pattern/1
  has discovery pattern: https://mif-spec.dev/ns/pattern/2

semantic
type: Namespace
has child namespace:
  contracts: contracts
  governance: governance
  storage: storage

contracts
type: Namespace
description: Data contracts and product interface agreements
type hint: semantic

governance
type: Namespace
description: Governance policies, controls, and stewardship
type hint: semantic

storage
type: Namespace
description: Storage and scaling architectures
type hint: semantic

procedural
type: Namespace
has child namespace:
  pipelines: pipelines

pipelines
type: Namespace
description: Pipeline and data-movement patterns
type hint: procedural

data-contract
type: EntityType
base type: SemanticMemory
description: A data contract: an enforceable agreement on schema, semantics, and SLAs between data producers and consumers
alternative label: data contract specification
alternative label: schema contract
alternative label: data contract policy
alternative label: interface contract
alternative label: data agreement
example: A contract-native data-contract specification enforces a required root-level field set validated as a pre-merge continuous-integration gate
example: A streaming-native data contract enumerates a fixed set of schema-compatibility modes and a fixed set of message-direction actions
example: A stewardship-overlay data contract adds an approval-state and ownership fields not present in purely technical enforcement contracts
negative example: Defines who may notify regulators or customers of a data incident and on what timeline, distinct from a required-reviewer rule scoped to specific file paths blocking merges to a shared table's schema.
negative example: A cross-functional group reviews new contracts before publication, distinct from querying an automated lineage catalog for every downstream table, dashboard, and feature depending on it before a structural change ships.
negative example: A platform team accepted semi-structured data into a lake's raw zone without enforcing schema at write time, deferring structure to the query layer, then enforcing it downstream in curated layers.
negative example: Contract-first development is an engineering practice where data structure and interface contracts are defined before implementation, ensuring producer and consumer teams agree on schema before either writes code.
negative example: The Confluent Schema Registry is a platform service that stores, versions, and serves Avro/Protobuf schemas for Kafka topics, enabling consumers to deserialize messages correctly across schema evolution.
negative example: ING's federated data mesh governance implements data contracts between domain teams where each domain publishes its data products with SLAs and versioned schemas, discoverable through a federated catalog.
defines trait: mif:trait/cited
schema.org:
  required: name
  properties:
    name:
      type: string
      description: Contract name or subject
    scope:
      type: string
      description: What the contract covers (dataset, interface, domain)
    parties:
      type: string
      description: Producer and consumer parties bound by the contract
    enforcement:
      type: string
      description: How the contract is enforced (CI gate, registry, runtime check)

data-product
type: EntityType
base type: SemanticMemory
description: A data product: a discoverable, governed, consumable data asset with an owner and interface
alternative label: data product
alternative label: data mesh product
alternative label: data-as-a-product asset
alternative label: productized dataset
example: An internal data-mesh project matured into a commercial cloud-native data marketplace serving institutional customers
example: A data mesh spanning many domains and hundreds of data products documented tens of millions in business benefit
example: A financial-technology company treats data as a product across several consumer-facing brands serving a large customer base
defines trait: mif:trait/cited
schema.org:
  required: name
  properties:
    name:
      type: string
      description: Data product name
    domain:
      type: string
      description: Owning business domain
    interface:
      type: string
      description: Consumption interface (view, table, API, share)
    sla:
      type: string
      description: Service-level agreement (freshness, availability)

data-governance-policy
type: EntityType
base type: SemanticMemory
description: A governance policy or control: a rule governing access, retention, classification, or stewardship of data
alternative label: access control policy
alternative label: retention policy
alternative label: data classification
alternative label: masking policy
alternative label: stewardship policy
alternative label: tagging policy
example: Row filters and column masks evaluated at query time via governed tags
example: Tiered storage lifecycle rules with regulation-aligned multi-year retention bands
example: A three-tier classification of sensitive records applied at the column level
example: Required at-birth documentation and ownership tags that hard-fail the build when missing
negative example: Requires data under residency obligations, like cross-border personal data transfer restrictions, be stored only in permitted jurisdiction regions, tracked per dataset rather than as one blanket warehouse-wide rule.
negative example: Saxo Bank implemented a data mesh where each business domain (e.g., trading, risk) manages and publishes its data product with documented ownership, quality standards, and access controls.
defines trait: mif:trait/cited
schema.org:
  required: policy_name
  properties:
    policy name:
      type: string
      description: Policy or control name
    scope:
      type: string
      description: What the policy governs (catalog, schema, table, tag)
    control:
      type: string
      description: Control type (preventive, detective, corrective)
    enforcement:
      type: string
      description: Mechanism that enforces the policy

data-quality-rule
type: EntityType
base type: SemanticMemory
description: A data quality rule, check, or assertion applied to data to detect or prevent defects
alternative label: data quality check
alternative label: validation rule
alternative label: data quality assertion
alternative label: quality gate rule
alternative label: data integrity rule
example: A sync-lag failure mode where a record exists in one system but is missing in a downstream system is already encoded as a test scenario
example: A reconciliation service level objective sets maximum freshness lag between two systems and a minimum availability percentage
example: A regression-test matrix for identity data covers several already-implemented failure scenarios plus gaps identified from real incidents
negative example: Tracks average time between a quality regression occurring and detection, separate from a single weighted score aggregating pass/fail checks where a critical rule's failure moves the score more than a low-severity one.
negative example: Maintains a small hand-curated dataset with verified expected outputs to catch regressions producing plausible but wrong values, distinct from validating as early as the raw-to-staging boundary rather than only at final tables.
defines trait: mif:trait/cited
schema.org:
  required: rule_name
  properties:
    rule name:
      type: string
      description: Rule or check name
    dimension:
      type: string
      description: Quality dimension (freshness, completeness, validity, uniqueness, drift)
    check:
      type: string
      description: How the check is expressed or computed
    action:
      type: string
      description: Reaction on violation (drop, mark, quarantine, alert, fail)

storage-architecture
type: EntityType
base type: SemanticMemory
description: A storage or scaling architecture for data (e.g. Postgres sharding, Snowflake multi-tenant isolation, lakehouse table format)
alternative label: data storage architecture
alternative label: database architecture
alternative label: storage topology
alternative label: data platform architecture
alternative label: storage design
example: A tier-based provisioning scheme encodes isolation level, compute sizing, and clone strategy per tenant tier in one declarative source
example: Multiple overlapping identity stores are joined by sync processes with no single declared system of record per attribute
example: A dual-path staging design splits an old and new export path, both required to remain consistent to prevent data-quality errors
example: Self-managed clustered database high availability requires explicit replication and failover configuration a managed cloud service would otherwise automate
negative example: A Databricks lakehouse team chose Delta Lake over plain Parquet files for ACID writes and time travel under concurrent writers, accepting the operational cost of periodic log compaction and vacuum maintenance.
negative example: A bring-your-own-key (BYOK) encryption policy requires that sensitive data be encrypted with customer-provided keys rather than service-managed keys, with key material never stored on the provider's infrastructure.
negative example: Snowflake's secure data sharing feature provides a data contract between provider and consumer where data stays in the provider's account but is accessible to authorized consumers with agreed-upon data governance terms.
defines trait: mif:trait/cited
schema.org:
  required: name
  properties:
    name:
      type: string
      description: Architecture name
    pattern:
      type: string
      description: The structural pattern (proxy, extension, shared/isolated, branching)
    scaling model:
      type: string
      description: How it scales (sharding, read replicas, serverless, multi-cluster)
    tradeoffs:
      type: string
      description: Key trade-offs and constraints

pipeline-pattern
type: EntityType
base type: ProceduralMemory
description: A pipeline or data-movement pattern (e.g. ETL/ELT, streaming, write-audit-publish, reverse-ETL)
alternative label: data pipeline pattern
alternative label: pipeline architecture
alternative label: ingestion pattern
alternative label: data-flow pattern
alternative label: processing pipeline
example: A five-stage ingest-profile-validate-observe-remediate pipeline is the interlock spine every other enforcement pattern maps onto
example: Contract-native enforcement places the quality gate at pre-merge continuous integration using a declarative data contract
example: Streaming-native enforcement stages writes on an audit branch and only publishes them after data-quality checks pass
example: Stewardship-overlay enforcement routes proposed schema or contract changes through a human steward's approval before they take effect
negative example: Isolates a noisy source's ingestion job onto its own dedicated compute pool so it can't starve others, distinct from an orchestrator dynamically fanning out per-partition parallel tasks based on partitions discovered at runtime.
negative example: The Airflow Scheduler parses DAGs and hands eligible task instances to an executor, distinct from a Debezium Kafka Connect connector reading one source database's transaction log and publishing change events per table.
negative example: Tracks the gap between now and the most recently landed data against a staleness threshold, distinct from mean time to recovery: the average elapsed time between a run failing and next producing correct output.
negative example: A team replaced nightly batch extraction with continuous change-data-capture streaming reading the source transaction log to cut load and staleness, distinct from an ELT approach loading raw data first and transforming later with warehouse SQL.
negative example: A launch-readiness gate blocks a new workflow from production until it has a test suite, named on-call owner, freshness targets, and rollback procedure, distinct from a rotating on-call responder with a defined escalation path.
negative example: Analytics engineering is an engineering practice where data engineers build transformations and models that prepare raw data for consumption by analysts and business intelligence tools, treating data as a product.
negative example: Apache Atlas with Hive lineage tracking records the provenance of data assets as they flow through transformations, allowing analysts to trace back why a metric has a particular value to the source systems that populated it.
negative example: Apache Airflow is a data orchestration platform that schedules and monitors workflows, Kafka is a streaming platform for event ingestion, and Fivetran is a managed ELT platform - each provides infrastructure for moving data, not just a single pattern.
negative example: Segment's Protocols is a tracking plan system where event schemas and required fields are defined centrally, creating a contract between the analytics library and the backend about what data will be sent.
defines trait: mif:trait/cited
schema.org:
  required: pattern_name
  properties:
    pattern name:
      type: string
      description: Pattern name
    context:
      type: string
      description: When to use this pattern
    solution:
      type: string
      description: How the pattern moves or validates data

data-platform
type: EntityType
base type: SemanticMemory
description: A named data platform, tool, catalog, or engine (e.g. Snowflake, Postgres, dbt, Airflow, Kafka, Iceberg, Unity Catalog)
alternative label: data warehouse
alternative label: data catalog
alternative label: data quality tool
alternative label: lakehouse engine
alternative label: streaming platform
alternative label: data tooling
example: An open-source data quality framework with declarative check definitions
example: A cloud data warehouse feature reaching general availability with native test integration
example: A catalog offering branch-like transactional versioning across multiple tables
example: A contract validation tool that emits pipeline annotations and diffs versions for breaking changes
negative example: A single Docker-packaged connector implements a fixed sync protocol for one source or destination, distinct from a Kafka broker process storing topic partitions, serving produce/fetch requests, and replicating via a controller quorum.
negative example: A large-scale data platform team migrated lake tables from the older table-tracking model to a newer snapshot-based metadata format for atomic commits and safe schema evolution under concurrent writers at petabyte scale.
negative example: Unity Catalog's automatic lineage feature traces data transformations across tables, capturing which source datasets feed into derived datasets and making that lineage queryable for impact analysis.
negative example: Airbnb's Minerva is a centralized metrics layer where metric definitions (like gross booking value) are authored once against governed tables and then automatically materialized for consumption by dashboards and analysis tools.
negative example: Apache Iceberg is a table format (not a platform) that adds ACID semantics and time-travel queries to data lakes, enabling efficient schema evolution and partition evolution without rewriting table data.
defines trait: mif:trait/cited
schema.org:
  required: name
  required: category
  properties:
    name:
      type: string
      description: Platform or tool name
    category:
      type: string
      description: Free-form category (warehouse, catalog, observability, streaming, proxy, table-format, transformation, etc.)
    vendor:
      type: string
      description: Maintaining vendor or project
    use case:
      type: string
      description: Primary role in the data stack

cited
type: Trait
description: Requires citations/sources
requires field: citations

enforces
type: Relationship
description: A governance policy or contract enforces a quality rule
domain: mif:entityType/data-governance-policy
domain: mif:entityType/data-contract
range: mif:entityType/data-quality-rule
symmetric property: false

governs
type: Relationship
description: A policy or platform governs a data product or architecture
domain: mif:entityType/data-governance-policy
domain: mif:entityType/data-platform
range: mif:entityType/data-product
range: mif:entityType/storage-architecture
symmetric property: false

implements
type: Relationship
description: A platform or architecture implements a pattern or decision
domain: mif:entityType/data-platform
domain: mif:entityType/storage-architecture
range: mif:entityType/pipeline-pattern
range: mif:entityType/design-pattern
range: mif:entityType/architectural-decision
symmetric property: false

governed_by
type: Relationship
description: A data product or storage architecture is governed by a control or policy
domain: mif:entityType/data-product
domain: mif:entityType/storage-architecture
range: mif:entityType/control
range: mif:entityType/policy
symmetric property: false

https://mif-spec.dev/ns/pattern/0
type: DiscoveryPattern
content pattern: \b(Snowflake|Postgres|PostgreSQL|BigQuery|Redshift)\b
suggests entity type: data-platform
suggests namespace: _semantic/entities

https://mif-spec.dev/ns/pattern/1
type: DiscoveryPattern
content pattern: \b(dbt|Airflow|Kafka|Iceberg|Delta\s+Lake|Unity\s+Catalog|Polaris)\b
suggests entity type: data-platform
suggests namespace: _semantic/entities

https://mif-spec.dev/ns/pattern/2
type: DiscoveryPattern
content pattern: \b(write-audit-publish|WAP|reverse-ETL|ELT|ETL)\b
suggests entity type: pipeline-pattern
suggests namespace: _procedural/patterns