PSDK-95: Enable schemata parameter in ScanRequest

gooddata-sdk/gooddata_sdk/catalog/data_source/action_requests/scan_model_request.py

@@ -22,6 +22,7 @@ class CatalogScanModelRequest(Base):
     scan_views: bool = field(default=False, validator=one_scan_true)
     table_prefix: Optional[str] = None
     view_prefix: Optional[str] = None
+    schemata: Optional[list[str]] = None
 
     @staticmethod
     def client_class() -> Type[ScanRequest]:

gooddata-sdk/gooddata_sdk/utils.py

@@ -6,6 +6,7 @@
 import re
 import typing
 from pathlib import Path
+from shutil import rmtree
 from typing import Any, Callable, Dict, NamedTuple, Union, cast
 
 import yaml
@@ -144,6 +145,12 @@ def create_directory(path: Path) -> None:
         os.makedirs(path)
 
 
+def recreate_directory(path: Path) -> None:
+    if os.path.exists(path):
+        rmtree(path)
+    os.makedirs(path)
+
+
 class IndentDumper(yaml.SafeDumper):
     @typing.no_type_check
     def increase_indent(self, flow: bool = False, indentless: bool = False):

gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_scan_model_with_schemata.yaml

@@ -0,0 +1,202 @@
+# (C) 2022 GoodData Corporation
+version: 1
+interactions:
+  - request:
+      method: POST
+      uri: http://localhost:3000/api/v1/actions/dataSources/demo-test-ds/scan
+      body:
+        scanTables: true
+        scanViews: false
+        separator: __
+        schemata:
+          - demo
+      headers:
+        Accept:
+          - application/json
+        Content-Type:
+          - application/json
+        X-GDC-VALIDATE-RELATIONS:
+          - 'true'
+        X-Requested-With:
+          - XMLHttpRequest
+    response:
+      status:
+        code: 200
+        message: OK
+      headers:
+        Access-Control-Allow-Credentials:
+          - 'true'
+        Access-Control-Expose-Headers:
+          - Content-Disposition, Content-Length, Content-Range, Set-Cookie
+        Cache-Control:
+          - no-cache, no-store, max-age=0, must-revalidate
+        Connection:
+          - keep-alive
+        Content-Length:
+          - '2372'
+        Content-Security-Policy:
+          - 'default-src ''self'' *.wistia.com *.wistia.net; script-src ''self'' ''unsafe-inline''
+            ''unsafe-eval'' *.wistia.com *.wistia.net src.litix.io matomo.anywhere.gooddata.com
+            code.jquery.com unpkg.com cdn.jsdelivr.net cdnjs.cloudflare.com; img-src
+            ''self'' data: blob: *.wistia.com *.wistia.net embedwistia-a.akamaihd.net
+            privacy-policy.truste.com www.gooddata.com; style-src ''self'' ''unsafe-inline''
+            fonts.googleapis.com cdn.jsdelivr.net fast.fonts.net; font-src ''self''
+            data: fonts.gstatic.com *.alicdn.com *.wistia.com cdn.jsdelivr.net info.gooddata.com;
+            frame-src ''self''; object-src ''none''; worker-src ''self'' blob:; child-src
+            blob:; connect-src ''self'' *.tiles.mapbox.com *.mapbox.com *.litix.io
+            *.wistia.com embedwistia-a.akamaihd.net matomo.anywhere.gooddata.com;
+            media-src ''self'' blob: data: *.wistia.com *.wistia.net embedwistia-a.akamaihd.net'
+        Content-Type:
+          - application/json
+        Date: &id001
+          - PLACEHOLDER
+        Expires:
+          - '0'
+        GoodData-Deployment:
+          - aio
+        Permission-Policy:
+          - geolocation 'none'; midi 'none'; sync-xhr 'none'; microphone 'none'; camera
+            'none'; magnetometer 'none'; gyroscope 'none'; fullscreen 'none'; payment
+            'none';
+        Pragma:
+          - no-cache
+        Referrer-Policy:
+          - no-referrer
+        Server:
+          - nginx
+        Set-Cookie:
+          - SPRING_REDIRECT_URI=; Path=/; Max-Age=0; Expires=Thu, 01 Jan 1970 00:00:00
+            GMT; HttpOnly; SameSite=Lax
+        Vary:
+          - Origin
+          - Access-Control-Request-Method
+          - Access-Control-Request-Headers
+        X-Content-Type-Options:
+          - nosniff
+        X-GDC-TRACE-ID: *id001
+        X-XSS-Protection:
+          - 1 ; mode=block
+      body:
+        string:
+          pdm:
+            tables:
+              - columns:
+                  - dataType: NUMERIC
+                    isPrimaryKey: false
+                    name: budget
+                  - dataType: STRING
+                    isPrimaryKey: true
+                    name: campaign_channel_id
+                  - dataType: INT
+                    isPrimaryKey: false
+                    name: campaign_id
+                    referencedTableColumn: campaign_id
+                    referencedTableId: campaigns
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: category
+                  - dataType: NUMERIC
+                    isPrimaryKey: false
+                    name: spend
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: type
+                id: campaign_channels
+                path:
+                  - demo
+                  - campaign_channels
+                type: TABLE
+              - columns:
+                  - dataType: INT
+                    isPrimaryKey: true
+                    name: campaign_id
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: campaign_name
+                id: campaigns
+                path:
+                  - demo
+                  - campaigns
+                type: TABLE
+              - columns:
+                  - dataType: INT
+                    isPrimaryKey: true
+                    name: customer_id
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: customer_name
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: geo__state__location
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: region
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: state
+                id: customers
+                path:
+                  - demo
+                  - customers
+                type: TABLE
+              - columns:
+                  - dataType: INT
+                    isPrimaryKey: false
+                    name: campaign_id
+                    referencedTableColumn: campaign_id
+                    referencedTableId: campaigns
+                  - dataType: INT
+                    isPrimaryKey: false
+                    name: customer_id
+                    referencedTableColumn: customer_id
+                    referencedTableId: customers
+                  - dataType: DATE
+                    isPrimaryKey: false
+                    name: date
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: order_id
+                  - dataType: STRING
+                    isPrimaryKey: true
+                    name: order_line_id
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: order_status
+                  - dataType: NUMERIC
+                    isPrimaryKey: false
+                    name: price
+                  - dataType: INT
+                    isPrimaryKey: false
+                    name: product_id
+                    referencedTableColumn: product_id
+                    referencedTableId: products
+                  - dataType: NUMERIC
+                    isPrimaryKey: false
+                    name: quantity
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: wdf__region
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: wdf__state
+                id: order_lines
+                path:
+                  - demo
+                  - order_lines
+                type: TABLE
+              - columns:
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: category
+                  - dataType: INT
+                    isPrimaryKey: true
+                    name: product_id
+                  - dataType: STRING
+                    isPrimaryKey: false
+                    name: product_name
+                id: products
+                path:
+                  - demo
+                  - products
+                type: TABLE
+          warnings: []

gooddata-sdk/tests/catalog/test_catalog_data_source.py

@@ -476,6 +476,16 @@ def test_scan_mode_with_table_prefix(test_config):
     assert scan_result.pdm.tables[0].name_prefix == "order"
 
 
+@gd_vcr.use_cassette(str(_fixtures_dir / "demo_test_scan_model_with_schemata.yaml"))
+def test_scan_mode_with_schemata(test_config):
+    sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
+    data_source_id = test_config["data_source"]
+    request = CatalogScanModelRequest(schemata=["demo"])
+
+    scan_result = sdk.catalog_data_source.scan_data_source(data_source_id, request)
+    assert len(scan_result.pdm.tables) == 5
+
+
 @gd_vcr.use_cassette(str(_fixtures_dir / "demo_test_scan_and_put_declarative_pdm.yaml"))
 def test_scan_and_put_declarative_pdm(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])

gooddata-sdk/tests/catalog/test_catalog_user_service.py

@@ -18,7 +18,7 @@
     GoodDataApiClient,
     GoodDataSdk,
 )
-from gooddata_sdk.utils import create_directory
+from gooddata_sdk.utils import recreate_directory
 
 gd_vcr = get_vcr()
 
@@ -168,7 +168,7 @@ def test_get_declarative_users(test_config):
 def test_store_declarative_users(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store"
-    create_directory(path)
+    recreate_directory(path)
 
     users_e = sdk.catalog_user.get_declarative_users()
     _assert_users_default(users_e.users)
@@ -234,7 +234,7 @@ def test_get_declarative_user_groups(test_config):
 def test_store_declarative_user_groups(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store"
-    create_directory(path)
+    recreate_directory(path)
 
     user_groups_e = sdk.catalog_user.get_declarative_user_groups()
     _assert_user_groups_default(user_groups_e.user_groups)
@@ -320,7 +320,7 @@ def test_get_declarative_users_user_groups(test_config):
 def test_store_declarative_users_user_groups(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store"
-    create_directory(path)
+    recreate_directory(path)
 
     users_user_groups_e = sdk.catalog_user.get_declarative_users_user_groups()
     _assert_users_user_groups_default(users_user_groups_e)

gooddata-sdk/tests/catalog/test_catalog_workspace.py

@@ -15,7 +15,7 @@
     GoodDataApiClient,
     GoodDataSdk,
 )
-from gooddata_sdk.utils import create_directory
+from gooddata_sdk.utils import recreate_directory
 
 gd_vcr = get_vcr()
 
@@ -79,7 +79,7 @@ def test_load_and_put_declarative_workspaces(test_config):
 def test_store_declarative_workspaces(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store"
-    create_directory(path)
+    recreate_directory(path)
 
     workspaces_e = sdk.catalog_workspace.get_declarative_workspaces()
     sdk.catalog_workspace.store_declarative_workspaces(path)
@@ -354,7 +354,7 @@ def test_get_declarative_workspace_data_filters(test_config):
 def test_store_declarative_workspace_data_filters(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store"
-    create_directory(path)
+    recreate_directory(path)
 
     declarative_workspace_data_filters_e = sdk.catalog_workspace.get_declarative_workspace_data_filters()
     sdk.catalog_workspace.store_declarative_workspace_data_filters(path)
@@ -455,7 +455,7 @@ def test_put_declarative_workspace(test_config):
 def test_store_declarative_workspace(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store"
-    create_directory(path)
+    recreate_directory(path)
 
     workspaces_e = sdk.catalog_workspace.get_declarative_workspace(workspace_id=test_config["workspace"])
     sdk.catalog_workspace.store_declarative_workspace(workspace_id=test_config["workspace"], layout_root_path=path)

gooddata-sdk/tests/catalog/test_catalog_workspace_content.py

@@ -16,7 +16,7 @@
     DataSourceValidator,
     GoodDataSdk,
 )
-from gooddata_sdk.utils import create_directory
+from gooddata_sdk.utils import recreate_directory
 
 gd_vcr = get_vcr()
 
@@ -65,7 +65,7 @@ def test_store_declarative_ldm(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store" / "workspace_content"
     workspace_id = test_config["workspace"]
-    create_directory(path)
+    recreate_directory(path)
 
     ldm_e = sdk.catalog_workspace_content.get_declarative_ldm(workspace_id)
     sdk.catalog_workspace_content.store_declarative_ldm(workspace_id, path)
@@ -135,7 +135,7 @@ def test_store_declarative_analytics_model(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     path = _current_dir / "store" / "workspace_content"
     workspace_id = test_config["workspace"]
-    create_directory(path)
+    recreate_directory(path)
 
     analytics_model_e = sdk.catalog_workspace_content.get_declarative_analytics_model(workspace_id)
     sdk.catalog_workspace_content.store_declarative_analytics_model(workspace_id, path)