From 43727a1ff89c88d82dd41596f8cee3532cb6247c Mon Sep 17 00:00:00 2001 From: Jacek Date: Wed, 14 Sep 2022 08:49:25 +0200 Subject: [PATCH 1/2] PSDK-95: Enable schemata parameter in ScanRequest --- .../action_requests/scan_model_request.py | 1 + .../demo_test_scan_model_with_schemata.yaml | 202 ++++++++++++++++++ .../tests/catalog/test_catalog_data_source.py | 10 + 3 files changed, 213 insertions(+) create mode 100644 gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_scan_model_with_schemata.yaml diff --git a/gooddata-sdk/gooddata_sdk/catalog/data_source/action_requests/scan_model_request.py b/gooddata-sdk/gooddata_sdk/catalog/data_source/action_requests/scan_model_request.py index cc724ee77..6812892fe 100644 --- a/gooddata-sdk/gooddata_sdk/catalog/data_source/action_requests/scan_model_request.py +++ b/gooddata-sdk/gooddata_sdk/catalog/data_source/action_requests/scan_model_request.py @@ -22,6 +22,7 @@ class CatalogScanModelRequest(Base): scan_views: bool = field(default=False, validator=one_scan_true) table_prefix: Optional[str] = None view_prefix: Optional[str] = None + schemata: Optional[list[str]] = None @staticmethod def client_class() -> Type[ScanRequest]: diff --git a/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_scan_model_with_schemata.yaml b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_scan_model_with_schemata.yaml new file mode 100644 index 000000000..ebccb6327 --- /dev/null +++ b/gooddata-sdk/tests/catalog/fixtures/data_sources/demo_test_scan_model_with_schemata.yaml @@ -0,0 +1,202 @@ +# (C) 2022 GoodData Corporation +version: 1 +interactions: + - request: + method: POST + uri: http://localhost:3000/api/v1/actions/dataSources/demo-test-ds/scan + body: + scanTables: true + scanViews: false + separator: __ + schemata: + - demo + headers: + Accept: + - application/json + Content-Type: + - application/json + X-GDC-VALIDATE-RELATIONS: + - 'true' + X-Requested-With: + - XMLHttpRequest + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - Content-Disposition, Content-Length, Content-Range, Set-Cookie + Cache-Control: + - no-cache, no-store, max-age=0, must-revalidate + Connection: + - keep-alive + Content-Length: + - '2372' + Content-Security-Policy: + - 'default-src ''self'' *.wistia.com *.wistia.net; script-src ''self'' ''unsafe-inline'' + ''unsafe-eval'' *.wistia.com *.wistia.net src.litix.io matomo.anywhere.gooddata.com + code.jquery.com unpkg.com cdn.jsdelivr.net cdnjs.cloudflare.com; img-src + ''self'' data: blob: *.wistia.com *.wistia.net embedwistia-a.akamaihd.net + privacy-policy.truste.com www.gooddata.com; style-src ''self'' ''unsafe-inline'' + fonts.googleapis.com cdn.jsdelivr.net fast.fonts.net; font-src ''self'' + data: fonts.gstatic.com *.alicdn.com *.wistia.com cdn.jsdelivr.net info.gooddata.com; + frame-src ''self''; object-src ''none''; worker-src ''self'' blob:; child-src + blob:; connect-src ''self'' *.tiles.mapbox.com *.mapbox.com *.litix.io + *.wistia.com embedwistia-a.akamaihd.net matomo.anywhere.gooddata.com; + media-src ''self'' blob: data: *.wistia.com *.wistia.net embedwistia-a.akamaihd.net' + Content-Type: + - application/json + Date: &id001 + - PLACEHOLDER + Expires: + - '0' + GoodData-Deployment: + - aio + Permission-Policy: + - geolocation 'none'; midi 'none'; sync-xhr 'none'; microphone 'none'; camera + 'none'; magnetometer 'none'; gyroscope 'none'; fullscreen 'none'; payment + 'none'; + Pragma: + - no-cache + Referrer-Policy: + - no-referrer + Server: + - nginx + Set-Cookie: + - SPRING_REDIRECT_URI=; Path=/; Max-Age=0; Expires=Thu, 01 Jan 1970 00:00:00 + GMT; HttpOnly; SameSite=Lax + Vary: + - Origin + - Access-Control-Request-Method + - Access-Control-Request-Headers + X-Content-Type-Options: + - nosniff + X-GDC-TRACE-ID: *id001 + X-XSS-Protection: + - 1 ; mode=block + body: + string: + pdm: + tables: + - columns: + - dataType: NUMERIC + isPrimaryKey: false + name: budget + - dataType: STRING + isPrimaryKey: true + name: campaign_channel_id + - dataType: INT + isPrimaryKey: false + name: campaign_id + referencedTableColumn: campaign_id + referencedTableId: campaigns + - dataType: STRING + isPrimaryKey: false + name: category + - dataType: NUMERIC + isPrimaryKey: false + name: spend + - dataType: STRING + isPrimaryKey: false + name: type + id: campaign_channels + path: + - demo + - campaign_channels + type: TABLE + - columns: + - dataType: INT + isPrimaryKey: true + name: campaign_id + - dataType: STRING + isPrimaryKey: false + name: campaign_name + id: campaigns + path: + - demo + - campaigns + type: TABLE + - columns: + - dataType: INT + isPrimaryKey: true + name: customer_id + - dataType: STRING + isPrimaryKey: false + name: customer_name + - dataType: STRING + isPrimaryKey: false + name: geo__state__location + - dataType: STRING + isPrimaryKey: false + name: region + - dataType: STRING + isPrimaryKey: false + name: state + id: customers + path: + - demo + - customers + type: TABLE + - columns: + - dataType: INT + isPrimaryKey: false + name: campaign_id + referencedTableColumn: campaign_id + referencedTableId: campaigns + - dataType: INT + isPrimaryKey: false + name: customer_id + referencedTableColumn: customer_id + referencedTableId: customers + - dataType: DATE + isPrimaryKey: false + name: date + - dataType: STRING + isPrimaryKey: false + name: order_id + - dataType: STRING + isPrimaryKey: true + name: order_line_id + - dataType: STRING + isPrimaryKey: false + name: order_status + - dataType: NUMERIC + isPrimaryKey: false + name: price + - dataType: INT + isPrimaryKey: false + name: product_id + referencedTableColumn: product_id + referencedTableId: products + - dataType: NUMERIC + isPrimaryKey: false + name: quantity + - dataType: STRING + isPrimaryKey: false + name: wdf__region + - dataType: STRING + isPrimaryKey: false + name: wdf__state + id: order_lines + path: + - demo + - order_lines + type: TABLE + - columns: + - dataType: STRING + isPrimaryKey: false + name: category + - dataType: INT + isPrimaryKey: true + name: product_id + - dataType: STRING + isPrimaryKey: false + name: product_name + id: products + path: + - demo + - products + type: TABLE + warnings: [] diff --git a/gooddata-sdk/tests/catalog/test_catalog_data_source.py b/gooddata-sdk/tests/catalog/test_catalog_data_source.py index 5c393990c..c64dfe84b 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_data_source.py +++ b/gooddata-sdk/tests/catalog/test_catalog_data_source.py @@ -476,6 +476,16 @@ def test_scan_mode_with_table_prefix(test_config): assert scan_result.pdm.tables[0].name_prefix == "order" +@gd_vcr.use_cassette(str(_fixtures_dir / "demo_test_scan_model_with_schemata.yaml")) +def test_scan_mode_with_schemata(test_config): + sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) + data_source_id = test_config["data_source"] + request = CatalogScanModelRequest(schemata=["demo"]) + + scan_result = sdk.catalog_data_source.scan_data_source(data_source_id, request) + assert len(scan_result.pdm.tables) == 5 + + @gd_vcr.use_cassette(str(_fixtures_dir / "demo_test_scan_and_put_declarative_pdm.yaml")) def test_scan_and_put_declarative_pdm(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) From a4619b371643e88b29763f689cff2233a5389269 Mon Sep 17 00:00:00 2001 From: Jacek Date: Wed, 14 Sep 2022 10:18:36 +0200 Subject: [PATCH 2/2] Tests: recreate folders containing stored fixtures --- gooddata-sdk/gooddata_sdk/utils.py | 7 +++++++ gooddata-sdk/tests/catalog/test_catalog_user_service.py | 8 ++++---- gooddata-sdk/tests/catalog/test_catalog_workspace.py | 8 ++++---- .../tests/catalog/test_catalog_workspace_content.py | 6 +++--- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/gooddata-sdk/gooddata_sdk/utils.py b/gooddata-sdk/gooddata_sdk/utils.py index 09b5cd5ce..026839b52 100644 --- a/gooddata-sdk/gooddata_sdk/utils.py +++ b/gooddata-sdk/gooddata_sdk/utils.py @@ -6,6 +6,7 @@ import re import typing from pathlib import Path +from shutil import rmtree from typing import Any, Callable, Dict, NamedTuple, Union, cast import yaml @@ -144,6 +145,12 @@ def create_directory(path: Path) -> None: os.makedirs(path) +def recreate_directory(path: Path) -> None: + if os.path.exists(path): + rmtree(path) + os.makedirs(path) + + class IndentDumper(yaml.SafeDumper): @typing.no_type_check def increase_indent(self, flow: bool = False, indentless: bool = False): diff --git a/gooddata-sdk/tests/catalog/test_catalog_user_service.py b/gooddata-sdk/tests/catalog/test_catalog_user_service.py index 133127f05..b2342d2da 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_user_service.py +++ b/gooddata-sdk/tests/catalog/test_catalog_user_service.py @@ -18,7 +18,7 @@ GoodDataApiClient, GoodDataSdk, ) -from gooddata_sdk.utils import create_directory +from gooddata_sdk.utils import recreate_directory gd_vcr = get_vcr() @@ -168,7 +168,7 @@ def test_get_declarative_users(test_config): def test_store_declarative_users(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" - create_directory(path) + recreate_directory(path) users_e = sdk.catalog_user.get_declarative_users() _assert_users_default(users_e.users) @@ -234,7 +234,7 @@ def test_get_declarative_user_groups(test_config): def test_store_declarative_user_groups(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" - create_directory(path) + recreate_directory(path) user_groups_e = sdk.catalog_user.get_declarative_user_groups() _assert_user_groups_default(user_groups_e.user_groups) @@ -320,7 +320,7 @@ def test_get_declarative_users_user_groups(test_config): def test_store_declarative_users_user_groups(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" - create_directory(path) + recreate_directory(path) users_user_groups_e = sdk.catalog_user.get_declarative_users_user_groups() _assert_users_user_groups_default(users_user_groups_e) diff --git a/gooddata-sdk/tests/catalog/test_catalog_workspace.py b/gooddata-sdk/tests/catalog/test_catalog_workspace.py index 276d35a10..7e2632f3b 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_workspace.py +++ b/gooddata-sdk/tests/catalog/test_catalog_workspace.py @@ -15,7 +15,7 @@ GoodDataApiClient, GoodDataSdk, ) -from gooddata_sdk.utils import create_directory +from gooddata_sdk.utils import recreate_directory gd_vcr = get_vcr() @@ -79,7 +79,7 @@ def test_load_and_put_declarative_workspaces(test_config): def test_store_declarative_workspaces(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" - create_directory(path) + recreate_directory(path) workspaces_e = sdk.catalog_workspace.get_declarative_workspaces() sdk.catalog_workspace.store_declarative_workspaces(path) @@ -354,7 +354,7 @@ def test_get_declarative_workspace_data_filters(test_config): def test_store_declarative_workspace_data_filters(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" - create_directory(path) + recreate_directory(path) declarative_workspace_data_filters_e = sdk.catalog_workspace.get_declarative_workspace_data_filters() sdk.catalog_workspace.store_declarative_workspace_data_filters(path) @@ -455,7 +455,7 @@ def test_put_declarative_workspace(test_config): def test_store_declarative_workspace(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" - create_directory(path) + recreate_directory(path) workspaces_e = sdk.catalog_workspace.get_declarative_workspace(workspace_id=test_config["workspace"]) sdk.catalog_workspace.store_declarative_workspace(workspace_id=test_config["workspace"], layout_root_path=path) diff --git a/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py b/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py index bb6768af6..c7e992978 100644 --- a/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py +++ b/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py @@ -16,7 +16,7 @@ DataSourceValidator, GoodDataSdk, ) -from gooddata_sdk.utils import create_directory +from gooddata_sdk.utils import recreate_directory gd_vcr = get_vcr() @@ -65,7 +65,7 @@ def test_store_declarative_ldm(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" / "workspace_content" workspace_id = test_config["workspace"] - create_directory(path) + recreate_directory(path) ldm_e = sdk.catalog_workspace_content.get_declarative_ldm(workspace_id) sdk.catalog_workspace_content.store_declarative_ldm(workspace_id, path) @@ -135,7 +135,7 @@ def test_store_declarative_analytics_model(test_config): sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"]) path = _current_dir / "store" / "workspace_content" workspace_id = test_config["workspace"] - create_directory(path) + recreate_directory(path) analytics_model_e = sdk.catalog_workspace_content.get_declarative_analytics_model(workspace_id) sdk.catalog_workspace_content.store_declarative_analytics_model(workspace_id, path)