feat(app): unified Studio installation permissions for DAB and Marketplace

Makefile

@@ -387,15 +387,21 @@ app-check-cli: ## Verify the Databricks CLI meets the minimum version for deploy
 #
 # ONE-TIME prerequisite per catalog (the bundle does not manage the pre-existing
 # catalog, so it cannot grant catalog-level access): grant USE CATALOG on the
-# chosen catalog to the app SP, the task-runner SP, and ``account users``.
-# See app/DEPLOYMENT.md.
+# chosen catalog to the app SP (plus CREATE SCHEMA), the task-runner SP, and the
+# configured UC audience principal (the studio_user_group, or ``account users`` in
+# broad mode). See app/DEPLOYMENT.md.
 #
 # Usage: make app-deploy PROFILE=my-profile TARGET=dev
 #        make app-deploy PROFILE=my-profile TARGET=dev \
 #                        BUNDLE_VARS='--var=catalog_name=foo'
+#        make app-deploy PROFILE=my-profile TARGET=dev \
+#                        STUDIO_PREFIX=dqx_studio_acme STUDIO_USER_GROUP=data-team
+#        make app-deploy PROFILE=my-profile TARGET=dev STUDIO_USER_GROUP=users  # broad mode
 #
 # BUNDLE_VARS forwards arbitrary ``--var key=value`` arguments to ``bundle
-# deploy`` and ``bundle run``.
+# deploy`` and ``bundle run``. STUDIO_PREFIX sets the ``prefix`` variable (storage
+# names) and STUDIO_USER_GROUP sets ``studio_user_group``; an explicit --var in
+# BUNDLE_VARS always wins.
 #
 # FORCE=1 appends ``--force`` to ``bundle deploy``. Use it when the deploy
 # aborts because a resource was modified in the workspace UI since the last
@@ -406,11 +412,28 @@ app-check-cli: ## Verify the Databricks CLI meets the minimum version for deploy
 app-deploy: app-check-cli $(if $(filter release,$(TARGET)),,app-build) ## Deploy and start app; release target uses prebuilt tag (FORCE=1 to overwrite remote edits)
 	@test -n "$(PROFILE)" || (echo "Usage: make app-deploy PROFILE=<databricks-profile> TARGET=<bundle-target>"; exit 1)
 	@test -n "$(TARGET)" || (echo "Usage: make app-deploy PROFILE=<databricks-profile> TARGET=<bundle-target>"; exit 1)
-	cd app && databricks bundle deploy -p $(PROFILE) -t $(TARGET) $(if $(FORCE),--force) $(BUNDLE_VARS)
-	cd app && databricks bundle run $(APP_NAME) -p $(PROFILE) -t $(TARGET) $(BUNDLE_VARS)
+	cd app && databricks bundle deploy -p $(PROFILE) -t $(TARGET) $(if $(FORCE),--force) $(STUDIO_BUNDLE_VARS)
+	cd app && databricks bundle run $(APP_NAME) -p $(PROFILE) -t $(TARGET) $(STUDIO_BUNDLE_VARS)
 
 APP_NAME ?= dqx-studio
 
+# Studio storage prefix / audience. STUDIO_USER_GROUP=users selects broad mode:
+# workspace ACLs use `users`, UC grants use `account users`. An explicit
+# studio_user_group in BUNDLE_VARS also suppresses the broad UC principal.
+studio_prefix_var = $(if $(STUDIO_PREFIX),$(if $(findstring prefix=,$(BUNDLE_VARS)),,--var prefix=$(STUDIO_PREFIX)))
+studio_group_var = $(if $(STUDIO_USER_GROUP),$(if $(findstring studio_user_group,$(BUNDLE_VARS)),,--var studio_user_group=$(STUDIO_USER_GROUP)))
+studio_uc_var = $(if $(filter users,$(STUDIO_USER_GROUP)),$(if $(findstring studio_uc_principal,$(BUNDLE_VARS))$(findstring studio_user_group,$(BUNDLE_VARS)),,--var "studio_uc_principal=account users"))
+STUDIO_BUNDLE_VARS = $(studio_prefix_var) $(studio_group_var) $(studio_uc_var) $(BUNDLE_VARS)
+
+# The broad-mode keyword is exactly lowercase `users`; any other casing would be
+# deployed as a (non-existent) dedicated group, so reject it before deploying.
+studio_group_lower = $(subst U,u,$(subst S,s,$(subst E,e,$(subst R,r,$(STUDIO_USER_GROUP)))))
+ifneq ($(filter app-deploy,$(MAKECMDGOALS)),)
+ifneq ($(and $(filter users,$(studio_group_lower)),$(filter-out users,$(STUDIO_USER_GROUP))),)
+$(error Broad mode requires lowercase STUDIO_USER_GROUP=users)
+endif
+endif
+
 ##@ Build & lockfiles
 
 build: ## Build sdist + wheel (with --require-hashes against build-constraints)

app/AGENTS.md

@@ -68,7 +68,7 @@ but is protected transitively by the instance-level guard.
 
 ```
 {user_catalog}
- ├── dqx_studio                       ← main schema (SP-managed)
+ ├── <prefix>                         ← main schema (default prefix `dqx_studio`; SP-managed)
  │   ├── dq_profiling_results         (Delta) profiler run results
  │   ├── dq_validation_runs           (Delta) dryrun + scheduled run history
  │   ├── dq_quarantine_records        (Delta) invalid rows captured by runs
@@ -84,8 +84,10 @@ but is protected transitively by the instance-level guard.
  │   ├── dq_schedule_configs_history  (OLTP*) schedule config change audit log
  │   ├── dq_schedule_runs             (OLTP*) scheduler last/next run state (survives restarts)
  │   └── dq_migrations                (Delta) Delta migration version tracker
- ├── dqx_studio_tmp                   ← temp views created via OBO for profiler/dryrun jobs
- └── dqx_studio.wheels (volume)       ← DQX + task-runner wheels uploaded at app startup
+ │   └── wheels (volume)              ← DQX + task-runner wheels uploaded at app startup
+ ├── <prefix>_tmp                     ← temp views created via OBO for profiler/dryrun jobs
+ ├── <prefix>_genie                   ← approved Genie views + metadata dimensions
+ └── <prefix>_demo                    ← demo source tables
 
 Lakebase project (when enabled, default `lakebase_project_id` = `dqx-studio-db`):
  └── databricks_postgres              (database — always-present admin DB; no per-app DB provisioned)
@@ -96,12 +98,16 @@ Lakebase project (when enabled, default `lakebase_project_id` = `dqx-studio-db`)
          └── dq_migrations             (Postgres migration version tracker)
 ```
 
-`(OLTP*)` = lives in **Lakebase Postgres**.
+The Lakebase schema (`DQX_LAKEBASE_SCHEMA`) never follows the UC prefix. UC storage names derive from an existing catalog and a validated prefix (`setup/storage.py`): DAB passes `DQX_CATALOG` / `DQX_PREFIX` (plus per-schema overrides) and treats them as authoritative; Marketplace has no volume binding and collects catalog, prefix, and audience through the setup form (`routes/v1/setup.py`, persisted via `setup/configuration.py`, locked once storage exists).
+
+`(OLTP*)` = lives in **Lakebase Postgres** when
+`lakebase_endpoint` is set, otherwise **Delta** (the
+`v2: Delta OLTP fallback` migration).
 
 ## Key Decisions
 
 - **No config.yaml** — all settings stored in Delta or Lakebase tables.
-- **Dedicated catalog** — user selects at install; `dqx_studio` and `dqx_studio_tmp` schemas are declared as bundle resources and created by `databricks bundle deploy`.
+- **Dedicated catalog** — user selects at install; the prefix-derived schemas (`<prefix>`, `<prefix>_tmp`, `<prefix>_genie`, `<prefix>_demo`) are declared as bundle resources and created by `databricks bundle deploy`; Marketplace setup creates them instead.
 - **Hybrid storage** — high-volume append tables in Delta; transactional/low-latency tables in Lakebase Postgres.
 - **Rule promotion** — export rules then deploy separately to prod; or save directly to prod checks table.
 - **Target environments** — Dev, UAT/QA (prod-like data); app is not intended for production rule execution.
@@ -110,16 +116,18 @@ Lakebase project (when enabled, default `lakebase_project_id` = `dqx-studio-db`)
 
 Stateful resources declared in `databricks.yml`:
 
-- `resources.schemas.main_schema` — `dqx_studio` schema
-- `resources.schemas.tmp_schema` — `dqx_studio_tmp` schema
-- `resources.volumes.wheels` — wheels volume
+- `resources.schemas.main_schema` — `${var.prefix}` schema
+- `resources.schemas.tmp_schema` — `${var.prefix}_tmp` schema
+- `resources.schemas.genie_schema` — `${var.prefix}_genie` schema
+- `resources.schemas.demo_schema` — `${var.prefix}_demo` schema
+- `resources.volumes.wheels` — `wheels` volume inside the main schema
 - `resources.postgres_projects.dqx_studio` — Lakebase Postgres project (autoscaling, scale-to-zero)
 
 Each carries `lifecycle.prevent_destroy: true` (Databricks CLI 0.268+), which blocks `databricks bundle destroy` and any deploy that would force-replace the resource. To intentionally tear something down: drop the flag, `databricks bundle deployment unbind <key> -t <target>`, then destroy.
 
 The app connects to the always-present `databricks_postgres` admin database on the Lakebase project (set as the default `lakebase_database_name`) via the `DQX_LAKEBASE_ENDPOINT` endpoint path and creates its own `dqx_studio` Postgres schema there on first start. The app SP's Postgres role (`resources.postgres_roles.app_sp`, a `DATABRICKS_SUPERUSER` member) grants the CREATE-schema privilege. We deliberately do not use `database_catalogs` because it also creates a Unity Catalog catalog and therefore requires `CREATE CATALOG` on the metastore — a permission most app deployers don't hold.
 
-UC privileges for the app SP and task-runner SP are declared **natively** as `grants:` on the schema/volume resources (using `${resources.apps.dqx-studio.service_principal_client_id}` and `${var.dqx_service_principal_application_id}`), so `databricks bundle deploy` applies them — there is no post-deploy grant script. The one exception is `USE CATALOG` on the pre-existing (user-selected) catalog, which the bundle can't grant because it doesn't manage the catalog; grant it once per catalog as a documented prerequisite (see `DEPLOYMENT.md`).
+UC privileges for the app SP and task-runner SP are declared **natively** as `grants:` on the schema/volume resources (using `${resources.apps.dqx-studio.service_principal_client_id}` and `${var.dqx_service_principal_application_id}`), so `databricks bundle deploy` applies them — there is no post-deploy grant script. DAB schemas are owned by the deployer, so the app SP gets an explicit privilege list **including `MANAGE`** on each Studio schema (`USE_SCHEMA`, `CREATE_TABLE`, `CREATE_FUNCTION`, `CREATE_VOLUME`, `SELECT`, `MODIFY`, `EXECUTE`, `READ_VOLUME`, `WRITE_VOLUME`, `APPLY_TAG`, `MANAGE`; never `ALL_PRIVILEGES`, because the bundle engine drops `MANAGE` when combined with it). The runner SP is least privilege: `USE_SCHEMA`/`SELECT`/`MODIFY` on the main schema, `USE_SCHEMA`/`SELECT` on `_tmp` (the runner reads OBO temp views through schema-level `SELECT`, so it can read any view in `_tmp`; there are no per-view runner grants, only the app SP's per-view `MANAGE` for cleanup), `READ_VOLUME` on wheels, nothing on `_genie`/`_demo`. The audience is granted via `studio_uc_principal` (the `studio_user_group`, or `account users` in broad mode); admin-group UC grants are deliberately not declared because `admin_group` may be the built-in `admins` (never a valid UC grantee), so the app applies and verifies them at runtime after each restart. The one manual step is catalog access on the pre-existing (user-selected) catalog, which the bundle can't grant because it doesn't manage the catalog: `USE CATALOG` + `CREATE SCHEMA` for the app SP, `USE CATALOG` for the runner and audience (see `DEPLOYMENT.md`). The warehouse ACL gives the app SP `CAN_MANAGE`; audience and admin group get `CAN_USE`.
 
 ## Architecture
 
@@ -496,7 +504,7 @@ See `DEVELOPMENT.md` for local `.env` and Lakebase notes.
 ## Important Notes
 
 - **SQL safety:** all interpolated identifiers must pass `validate_fqn` and be wrapped with `quote_fqn` from `sql_utils.py`. All string literals must be escaped with `escape_sql_string` (ANSI doubled quotes — never backslash). User-supplied SQL bodies must pass `is_sql_query_safe()` from the DQX library and raise `UnsafeSqlQueryError` on rejection.
-- **Setup and activation:** Lakebase and Delta migrations, app-SP sibling-schema capabilities, score views, and the entitlement view are required before the app reports ready. End-user grants are best effort and never gate readiness; catalog access can be scoped to the intended user groups. Genie SELECT grants are limited to five approved views and two metadata tables (`dim_dq_rules` and `dim_dq_monitored_tables`), never the whole schema or entitlement table. Each successful metadata refresh retries the metadata-table grants. Metadata refresh and Genie space provisioning remain best effort.
+- **Setup and activation:** Lakebase and Delta migrations, app-SP catalog/schema capabilities, score views, and the entitlement view are required before the app reports ready. Audience, administrator, and runner access is **applied and verified by the setup access step** (`setup/access.py`, `setup/checks.py`): setup attempts each grant or additive ACL update, then re-reads the state, and missing or uninspectable grants block readiness (fail closed). The one exception is app sharing (`app_sharing` step): if neither the app SP nor the setup administrator can read the app ACL it reports a warning, not a block. Configured audiences come from `setup/audience.py`: a dedicated group, or `users` in DAB broad mode only (workspace ACL `users`, UC `account users`); the workspace `admins` group never receives UC grants, only a custom `DQX_ADMIN_GROUP` does. Setup mutations are allowed for members of `DQX_ADMIN_GROUP` or `admins`. Genie SELECT grants are limited to five approved views and two metadata tables (`dim_dq_rules` and `dim_dq_monitored_tables`), never the whole schema or entitlement table. Each successful metadata refresh retries the metadata-table grants. The metadata-dimension refresh is required for activation (the access step needs the metadata tables); only Genie space provisioning remains best effort. ACL changes are additive (`update_permissions`), never `set_permissions`.
 - **Scheduler:** runs in-process as an asyncio task, gated by an exclusive file lock (`/tmp/.dqx_scheduler.lock`) so only one uvicorn worker drives it. Disable with `DQX_SCHEDULER_DISABLED=1`.
 - **Caches:** `app_cache` (`cache.py`) is per-process in-memory with TTL. SP `WorkspaceClient`, OBO `WorkspaceClient`, and per-user catalog list are all cached. Use the `MISS` sentinel — never `is None` — to detect cache absence.
 - **SPA static files:** `spa_static.py` falls through to `index.html` only for non-asset paths (positive allowlist of asset extensions), so SPA routes containing dots still work.
@@ -576,9 +584,11 @@ to their native syntax.
 Stateful resources declared in `databricks.yml` with
 `lifecycle.prevent_destroy: true` (Databricks CLI 0.268+):
 
-* `resources.schemas.main_schema` — `dqx_studio` schema
-* `resources.schemas.tmp_schema` — `dqx_studio_tmp` schema
-* `resources.volumes.wheels` — wheels volume
+* `resources.schemas.main_schema` — `${var.prefix}` schema
+* `resources.schemas.tmp_schema` — `${var.prefix}_tmp` schema
+* `resources.schemas.genie_schema` — `${var.prefix}_genie` schema
+* `resources.schemas.demo_schema` — `${var.prefix}_demo` schema
+* `resources.volumes.wheels` — `wheels` volume inside the main schema
 * `resources.postgres_projects.dqx_studio` — Lakebase Postgres project
   (autoscaling + scale-to-zero per [Lakebase Autoscaling](https://docs.databricks.com/aws/en/oltp/upgrade-to-autoscaling)),
   paired with `resources.postgres_roles.app_sp` (the app SP's Postgres role)
@@ -615,8 +625,11 @@ UC privileges for the app SP and task-runner SP are declared
 **natively** as `grants:` on the schema/volume resources (via
 `${resources.apps.dqx-studio.service_principal_client_id}` and
 `${var.dqx_service_principal_application_id}`), so `bundle deploy`
-applies them — there is no post-deploy grant script. The one manual
-step is `USE CATALOG` on the pre-existing (user-selected) catalog,
-which the bundle can't grant because it doesn't manage the catalog;
-grant it once per catalog as a documented prerequisite (see
-`DEPLOYMENT.md`).
+applies them — there is no post-deploy grant script. The app SP gets
+an explicit privilege list including `MANAGE` (no `ALL_PRIVILEGES`) on the Studio schemas; the runner SP is
+least privilege (see "Bundle conventions" above). The one manual step
+is catalog access on the pre-existing (user-selected) catalog, which
+the bundle can't grant because it doesn't manage the catalog; grant it
+once per catalog as a documented prerequisite (see `DEPLOYMENT.md`).
+The wheels `uc_securable` app binding is DAB-only; the Marketplace
+manifest binds just the warehouse (`CAN_MANAGE`) and Lakebase.

app/CUSTOMER_QA.md

@@ -42,8 +42,8 @@ One Databricks Asset Bundle (`databricks.yml`) provisions everything in one `mak
 - **Databricks App** (FastAPI + React, single process, served by the Apps runtime)
 - **Serverless Job** for Spark work — the *task runner* — invoked for profiler, dry-run, and scheduled runs
 - **Lakebase Postgres project** (`postgres_projects.dqx_studio` + `postgres_roles.app_sp`) for OLTP state
-- **Two UC schemas** (`dqx_studio`, `dqx_studio_tmp`) under a customer-supplied catalog
-- **UC volume** (`wheels`) the app uses to ship DQX wheels into the job
+- **Four UC schemas** derived from a storage prefix (default `dqx_studio`): `<prefix>`, `<prefix>_tmp`, `<prefix>_genie`, `<prefix>_demo`, under a customer-supplied catalog
+- **UC volume** (`<prefix>.wheels`) the app uses to ship DQX wheels into the job
 - **SQL warehouse** — managed by the bundle, or BYO (bring-your-own) if you already have one
 - **Lakeview dashboard** (`dashboards.dqx_quality_overview`) pinned to the app's *Insights* page
 
@@ -154,15 +154,17 @@ Roles are resolved from **Databricks workspace-group membership** via the `dq_ro
 ### 4.3 What permissions does the app's service principal need?
 Scoped tight:
 
-- `USE CATALOG` + `USE SCHEMA` + `ALL PRIVILEGES` on the two DQX schemas (`dqx_studio`, `dqx_studio_tmp`) only
-- `READ VOLUME` + `WRITE VOLUME` on the wheels volume only
-- `CAN USE` on the SQL warehouse the app is bound to
+- `USE CATALOG` + `CREATE SCHEMA` on the selected catalog (the one manual prerequisite)
+- Ownership (Marketplace) or an explicit privilege list incl. `MANAGE` (DAB) on the four prefix-derived Studio schemas and the wheels volume only
+- `CAN MANAGE` on the SQL warehouse the app is bound to, so setup can share it with the audience additively
 - `Service Principal: User` role on the task-runner SP (so the app can submit jobs as it)
 
+The task-runner SP is least privilege: `USE CATALOG`; `USE SCHEMA`, `SELECT`, `MODIFY` on the main schema; `USE SCHEMA` and `SELECT` on `<prefix>_tmp` (the runner reads OBO temporary views through this schema-level `SELECT`, so it can read any view in `_tmp`; no per-view grants); `READ VOLUME` on the wheels volume; nothing on `<prefix>_genie` or `<prefix>_demo`.
+
 It is **not** a workspace admin and **not** a metastore admin. If you remove the app, those grants are the only blast radius.
 
 ### 4.4 What's the deployer's permission burden?
-Documented as a table in `DEPLOYMENT.md` — about 10 line items, the bulk of which collapse if the deployer is added to a UC-admin group. The single most common failure on first deploy is missing `MANAGE`/`USE CATALOG` on the target catalog. `bundle deploy` applies schema/volume grants natively, but the one manual prerequisite is granting `USE CATALOG` on the (pre-existing, bundle-unmanaged) catalog to the app SP, task-runner SP, and `account users`. We surface that error explicitly with a fix.
+Documented as a table in `DEPLOYMENT.md` — about 10 line items, the bulk of which collapse if the deployer is added to a UC-admin group. The single most common failure on first deploy is missing `MANAGE`/`USE CATALOG` on the target catalog. `bundle deploy` applies schema/volume grants natively, but the one manual prerequisite is granting catalog access on the (pre-existing, bundle-unmanaged) catalog: `USE CATALOG` + `CREATE SCHEMA` to the app SP, and `USE CATALOG` to the task-runner SP and the audience group (`account users` only in broad mode). We surface that error explicitly with a fix.
 
 ### 4.5 Is everything audited?
 Yes, but the auditing is **distributed** across the platform rather than in one DQX log: