Adaptive Data Pipelines

A workflow with a composite trigger (event + cron) plus within-DAG ordering.

from ematix_flow import ematix, ManagedTable, Annotated, BigInt, Text, pk

@ematix.connection
class warehouse:
    kind = "postgres"
    url = "${WAREHOUSE_URL}"

class OrdersExtracted(ManagedTable):
    __schema__ = "analytics"; __tablename__ = "orders_extracted"
    order_id: Annotated[BigInt, pk()]
    customer_id: BigInt
    amount_cents: BigInt

class OrdersEnriched(ManagedTable):
    __schema__ = "analytics"; __tablename__ = "orders_enriched"
    order_id: Annotated[BigInt, pk()]
    amount_bucket: Text

@ematix.job(name="extract_orders",
            target=OrdersExtracted, target_connection="warehouse",
            mode="merge", keys=("order_id",))
def extract_orders(conn):
    return "SELECT order_id, customer_id, amount_cents FROM raw.orders"

@ematix.job(name="enrich_orders",
            target=OrdersEnriched, target_connection="warehouse",
            mode="merge", keys=("order_id",),
            depends_on=["extract_orders"])
def enrich_orders(conn):
    return "SELECT order_id, CASE WHEN amount_cents < 10000 THEN 'small' ELSE 'large' END AS amount_bucket FROM analytics.orders_extracted"

# Workflow declares the trigger; member jobs declare their DAG position.
ematix.workflow(
    name="orders_etl",
    triggered_by=["upstream_workflow"],
    schedule="0 21 * * *",
    timezone="America/New_York",
    jobs=["extract_orders", "enrich_orders"],
)