diff --git a/.editorconfig b/.editorconfig index 8edc0526..ba04fbc3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,4 +9,4 @@ indent_size=2 trim_trailing_whitespace=true [*.py] -indent_size=4 \ No newline at end of file +indent_size=4 diff --git a/.github/workflows/dockerhub_push.yml b/.github/workflows/dockerhub_push.yml index 6cabb9cb..5913a7ce 100644 --- a/.github/workflows/dockerhub_push.yml +++ b/.github/workflows/dockerhub_push.yml @@ -40,7 +40,7 @@ jobs: tags: | type=ref,event=branch type=semver,pattern={{version}} - - + - name: Echo published tags run: | echo "Published docker tags: ${{ steps.meta.outputs.tags }}" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..b75d06ea --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [master, main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v2.0.3 diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..b9fb3f3e --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +profile=black diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..ac3e32bc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort diff --git a/Dockerfile b/Dockerfile index a8fde254..648d492c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,4 +72,4 @@ EXPOSE 7000 # expose opa directly EXPOSE 8181 # run gunicorn -CMD ["/start.sh"] \ No newline at end of file +CMD ["/start.sh"] diff --git a/MANIFEST.in b/MANIFEST.in index 990de4e4..fdf5fbe4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -recursive-include horizon/static * \ No newline at end of file +recursive-include horizon/static * diff --git a/README.md b/README.md index 7ae76a2b..e8db9185 100644 --- a/README.md +++ b/README.md @@ -36,4 +36,4 @@ you must declare the environment variable `READ_ONLY_GITHUB_TOKEN` for this comm ``` DEV_MODE_CLIENT_TOKEN= make run ``` -you must declare the environment variable `DEV_MODE_CLIENT_TOKEN` for this command to work. \ No newline at end of file +you must declare the environment variable `DEV_MODE_CLIENT_TOKEN` for this command to work. diff --git a/horizon/authentication.py b/horizon/authentication.py index 2e27baee..fb6a18fa 100644 --- a/horizon/authentication.py +++ b/horizon/authentication.py @@ -5,7 +5,9 @@ def enforce_pdp_token(authorization=Header(None)): if authorization is None: - raise HTTPException(status.HTTP_401_UNAUTHORIZED, detail="Missing Authorization header") + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, detail="Missing Authorization header" + ) schema, token = authorization.split(" ") if schema.strip().lower() != "bearer" or token.strip() != sidecar_config.API_KEY: diff --git a/horizon/config.py b/horizon/config.py index 7b760081..7e88e85a 100644 --- a/horizon/config.py +++ b/horizon/config.py @@ -4,11 +4,19 @@ class SidecarConfig(Confi): - CONTROL_PLANE = confi.str("CONTROL_PLANE", "http://localhost:8000", description="URL to the control plane that manages this PDP, typically Permit.io cloud (api.permit.io)") + CONTROL_PLANE = confi.str( + "CONTROL_PLANE", + "http://localhost:8000", + description="URL to the control plane that manages this PDP, typically Permit.io cloud (api.permit.io)", + ) # backend api url, where proxy requests go - BACKEND_SERVICE_URL = confi.str("BACKEND_SERVICE_URL", confi.delay("{CONTROL_PLANE}/v1")) - BACKEND_LEGACY_URL = confi.str("BACKEND_LEGACY_URL", confi.delay("{CONTROL_PLANE}/sdk")) + BACKEND_SERVICE_URL = confi.str( + "BACKEND_SERVICE_URL", confi.delay("{CONTROL_PLANE}/v1") + ) + BACKEND_LEGACY_URL = confi.str( + "BACKEND_LEGACY_URL", confi.delay("{CONTROL_PLANE}/sdk") + ) # backend route to fetch policy data topics REMOTE_CONFIG_ENDPOINT = confi.str("REMOTE_CONFIG_ENDPOINT", "pdps/me/config") @@ -26,23 +34,60 @@ class SidecarConfig(Confi): ENABLE_MONITORING = confi.bool("ENABLE_MONITORING", False) # centralized logging - CENTRAL_LOG_DRAIN_URL = confi.str("CENTRAL_LOG_DRAIN_URL", "https://listener.logz.io:8071") + CENTRAL_LOG_DRAIN_URL = confi.str( + "CENTRAL_LOG_DRAIN_URL", "https://listener.logz.io:8071" + ) CENTRAL_LOG_DRAIN_TIMEOUT = confi.int("CENTRAL_LOG_DRAIN_TIMEOUT", 5) CENTRAL_LOG_TOKEN = confi.str("CENTRAL_LOG_TOKEN", None) CENTRAL_LOG_ENABLED = confi.bool("CENTRAL_LOG_ENABLED", False) # internal OPA config - OPA_CONFIG_FILE_PATH = confi.str("OPA_CONFIG_FILE_PATH", "~/opa/config.yaml", description="the path on the container for OPA config file") - OPA_AUTH_POLICY_FILE_PATH = confi.str("OPA_AUTH_POLICY_FILE_PATH", "~/opa/basic-authz.rego", description="the path on the container for OPA authorization policy (rego file)") - OPA_BEARER_TOKEN_REQUIRED = confi.bool("OPA_BEARER_TOKEN_REQUIRED", True, description="if true, all API calls to OPA must provide a bearer token (the value of CLIENT_TOKEN)") - OPA_DECISION_LOG_ENABLED = confi.bool("OPA_DECISION_LOG_ENABLED", True, description="if true, OPA decision logs will be uploaded to the Permit.io cloud console") - OPA_DECISION_LOG_CONSOLE = confi.bool("OPA_DECISION_LOG_CONSOLE", False, description="if true, OPA decision logs will also be printed to console (only relevant if `OPA_DECISION_LOG_ENABLED` is true)") - OPA_DECISION_LOG_INGRESS_ROUTE = confi.str("OPA_DECISION_LOG_INGRESS_ROUTE", "/v1/decision_logs/ingress", description="the route on the backend the decision logs will be uploaded to") - OPA_DECISION_LOG_MIN_DELAY = confi.int("OPA_DECISION_LOG_MIN_DELAY", 1, description="min amount of time (in seconds) to wait between decision log uploads") - OPA_DECISION_LOG_MAX_DELAY = confi.int("OPA_DECISION_LOG_MAX_DELAY", 10, description="max amount of time (in seconds) to wait between decision log uploads") + OPA_CONFIG_FILE_PATH = confi.str( + "OPA_CONFIG_FILE_PATH", + "~/opa/config.yaml", + description="the path on the container for OPA config file", + ) + OPA_AUTH_POLICY_FILE_PATH = confi.str( + "OPA_AUTH_POLICY_FILE_PATH", + "~/opa/basic-authz.rego", + description="the path on the container for OPA authorization policy (rego file)", + ) + OPA_BEARER_TOKEN_REQUIRED = confi.bool( + "OPA_BEARER_TOKEN_REQUIRED", + True, + description="if true, all API calls to OPA must provide a bearer token (the value of CLIENT_TOKEN)", + ) + OPA_DECISION_LOG_ENABLED = confi.bool( + "OPA_DECISION_LOG_ENABLED", + True, + description="if true, OPA decision logs will be uploaded to the Permit.io cloud console", + ) + OPA_DECISION_LOG_CONSOLE = confi.bool( + "OPA_DECISION_LOG_CONSOLE", + False, + description="if true, OPA decision logs will also be printed to console (only relevant if `OPA_DECISION_LOG_ENABLED` is true)", + ) + OPA_DECISION_LOG_INGRESS_ROUTE = confi.str( + "OPA_DECISION_LOG_INGRESS_ROUTE", + "/v1/decision_logs/ingress", + description="the route on the backend the decision logs will be uploaded to", + ) + OPA_DECISION_LOG_MIN_DELAY = confi.int( + "OPA_DECISION_LOG_MIN_DELAY", + 1, + description="min amount of time (in seconds) to wait between decision log uploads", + ) + OPA_DECISION_LOG_MAX_DELAY = confi.int( + "OPA_DECISION_LOG_MAX_DELAY", + 10, + description="max amount of time (in seconds) to wait between decision log uploads", + ) # temp log format (until cloud config is received) - TEMP_LOG_FORMAT = confi.str("TEMP_LOG_FORMAT", "{time} | {process} | {name: <40}|{level:^6} | {message}") + TEMP_LOG_FORMAT = confi.str( + "TEMP_LOG_FORMAT", + "{time} | {process} | {name: <40}|{level:^6} | {message}", + ) # non configurable values ------------------------------------------------- @@ -50,31 +95,32 @@ class SidecarConfig(Confi): OPENAPI_TAGS_METADATA = [ { "name": "Authorization API", - "description": "Authorization queries to OPA. These queries are answered locally by OPA " + \ - "and do not require the cloud service. Latency should be very low (< 20ms per query)" + "description": "Authorization queries to OPA. These queries are answered locally by OPA " + + "and do not require the cloud service. Latency should be very low (< 20ms per query)", }, { "name": "Local Queries", - "description": "These queries are done locally against the sidecar and do not " + \ - "involve a network round-trip to Permit.io cloud API. Therefore they are safe " + \ - "to use with reasonable performance (i.e: with negligible latency) in the context of a user request.", + "description": "These queries are done locally against the sidecar and do not " + + "involve a network round-trip to Permit.io cloud API. Therefore they are safe " + + "to use with reasonable performance (i.e: with negligible latency) in the context of a user request.", }, { "name": "Policy Updater", - "description": "API to manually trigger and control the local policy caching and refetching." + "description": "API to manually trigger and control the local policy caching and refetching.", }, { "name": "Cloud API Proxy", - "description": "These endpoints proxy the Permit.io cloud api, and therefore **incur high-latency**. " + \ - "You should not use the cloud API in the standard request flow of users, i.e in places where the incurred " + \ - "added latency will affect your entire api. A good place to call the cloud API will be in one-time user events " + \ - "such as user registration (i.e: calling sync user, assigning initial user roles, etc.). " + \ - "The sidecar will proxy to the cloud every request prefixed with '/sdk'.", + "description": "These endpoints proxy the Permit.io cloud api, and therefore **incur high-latency**. " + + "You should not use the cloud API in the standard request flow of users, i.e in places where the incurred " + + "added latency will affect your entire api. A good place to call the cloud API will be in one-time user events " + + "such as user registration (i.e: calling sync user, assigning initial user roles, etc.). " + + "The sidecar will proxy to the cloud every request prefixed with '/sdk'.", "externalDocs": { "description": "The cloud api complete docs are located here:", "url": "https://api.permit.io/redoc", }, - } + }, ] -sidecar_config = SidecarConfig(prefix="PDP_") \ No newline at end of file + +sidecar_config = SidecarConfig(prefix="PDP_") diff --git a/horizon/enforcer/api.py b/horizon/enforcer/api.py index f3631c85..c5efaa53 100644 --- a/horizon/enforcer/api.py +++ b/horizon/enforcer/api.py @@ -1,17 +1,17 @@ import json -from typing import Optional, Dict +from typing import Dict, Optional -from fastapi import APIRouter, status, Response, Depends -from opal_client.policy_store import BasePolicyStoreClient, DEFAULT_POLICY_STORE_GETTER -from opal_client.policy_store.opa_client import fail_silently +from fastapi import APIRouter, Depends, Response, status from opal_client.logger import logger +from opal_client.policy_store import DEFAULT_POLICY_STORE_GETTER, BasePolicyStoreClient +from opal_client.policy_store.opa_client import fail_silently from horizon.authentication import enforce_pdp_token from horizon.config import sidecar_config - from horizon.enforcer.schemas import AuthorizationQuery, AuthorizationResult -def init_enforcer_api_router(policy_store:BasePolicyStoreClient=None): + +def init_enforcer_api_router(policy_store: BasePolicyStoreClient = None): policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() router = APIRouter(dependencies=[Depends(enforce_pdp_token)]) @@ -24,12 +24,20 @@ def log_query_and_result(query: AuthorizationQuery, response: Response): granting_role = None if allowed: granting_permissions = result.get("granting_permission", []) - granting_permission = {} if len(granting_permissions) == 0 else granting_permissions[0] + granting_permission = ( + {} if len(granting_permissions) == 0 else granting_permissions[0] + ) permission = granting_permission.get("permission", {}) - granting_role: Optional[Dict] = granting_permission.get("granting_role", None) + granting_role: Optional[Dict] = granting_permission.get( + "granting_role", None + ) if granting_role: role_id = granting_role.get("id", "__NO_ID__") - roles = [r for r in result.get("user_roles", []) if r.get("id", "") == role_id] + roles = [ + r + for r in result.get("user_roles", []) + if r.get("id", "") == role_id + ] granting_role = granting_role if not roles else roles[0] debug = { @@ -52,29 +60,36 @@ def log_query_and_result(query: AuthorizationQuery, response: Response): allowed=allowed, api_params=params, input=query.dict(), - debug=debug + debug=debug, ) except: try: body = str(response.body, "utf-8") except: body = None - data = {} if body is None else {"response_body" : body} - logger.info("is allowed", + data = {} if body is None else {"response_body": body} + logger.info( + "is allowed", params=params, query=query.dict(), response_status=response.status_code, **data ) - - @router.post("/allowed", response_model=AuthorizationResult, status_code=status.HTTP_200_OK, response_model_exclude_none=True) + @router.post( + "/allowed", + response_model=AuthorizationResult, + status_code=status.HTTP_200_OK, + response_model_exclude_none=True, + ) async def is_allowed(query: AuthorizationQuery): async def _is_allowed(): return await policy_store.get_data_with_input(path="rbac", input=query) fallback_response = dict(result=dict(allow=False, debug="OPA not responding")) - is_allowed_with_fallback = fail_silently(fallback=fallback_response)(_is_allowed) + is_allowed_with_fallback = fail_silently(fallback=fallback_response)( + _is_allowed + ) response = await is_allowed_with_fallback() log_query_and_result(query, response) try: @@ -82,23 +97,28 @@ async def _is_allowed(): processed_query = raw_result.get("authorization_query", {}) result = { "allow": raw_result.get("allow", False), - "result": raw_result.get("allow", False), # fallback for older sdks (TODO: remove) + "result": raw_result.get( + "allow", False + ), # fallback for older sdks (TODO: remove) "query": { "user": processed_query.get("user", {"id": query.user}), "action": processed_query.get("action", query.action), - "resource": processed_query.get("resource", query.resource.dict(exclude_none=True)), + "resource": processed_query.get( + "resource", query.resource.dict(exclude_none=True) + ), }, "debug": { "warnings": raw_result.get("debug", []), "user_roles": raw_result.get("user_roles", []), "granting_permission": raw_result.get("granting_permission", []), "user_permissions": raw_result.get("user_permissions", []), - } + }, } except: result = dict(allow=False, result=False) - logger.warning("is allowed (fallback response)", reason="cannot decode opa response") + logger.warning( + "is allowed (fallback response)", reason="cannot decode opa response" + ) return result return router - diff --git a/horizon/enforcer/opa/config_maker.py b/horizon/enforcer/opa/config_maker.py index d9e5cd11..8c3b72a7 100644 --- a/horizon/enforcer/opa/config_maker.py +++ b/horizon/enforcer/opa/config_maker.py @@ -1,7 +1,8 @@ import os -import jinja2 +import jinja2 from opal_common.logger import logger + from horizon.config import SidecarConfig @@ -17,7 +18,7 @@ def persist_to_file(contents: str, path: str) -> str: os.makedirs(os.path.dirname(path), exist_ok=True) # persist to file - with open(path, 'w') as f: + with open(path, "w") as f: f.write(contents) return path @@ -25,7 +26,7 @@ def persist_to_file(contents: str, path: str) -> str: def get_opa_config_file_path( sidecar_config: SidecarConfig, - template_path = "config.yaml.template", + template_path="config.yaml.template", ) -> str: """ renders a template that implements the OPA config file, according to the official spec: @@ -57,9 +58,10 @@ def get_opa_config_file_path( return persist_to_file(contents, target_path) + def get_opa_authz_policy_file_path( sidecar_config: SidecarConfig, - template_path = "authz.rego.template", + template_path="authz.rego.template", ) -> str: """ renders a template that implements a rego policy for OPA authz, as demonstrated here: diff --git a/horizon/enforcer/schemas.py b/horizon/enforcer/schemas.py index fbff05c1..e57b9989 100644 --- a/horizon/enforcer/schemas.py +++ b/horizon/enforcer/schemas.py @@ -1,4 +1,5 @@ -from typing import Dict, Any, Optional, List +from typing import Any, Dict, List, Optional + from pydantic import BaseModel @@ -19,24 +20,28 @@ class AuthorizationQuery(BaseSchema): """ the format of is_allowed() input """ - user: str # user_id or jwt + + user: str # user_id or jwt action: str resource: Resource context: Optional[Dict[str, Any]] = {} + class ProcessedQuery(BaseSchema): user: Dict[str, Any] action: str resource: Dict[str, Any] + class DebugInformation(BaseSchema): warnings: Optional[List[str]] user_roles: Optional[List[Dict[str, Any]]] granting_permission: Optional[List[Dict[str, Any]]] user_permissions: Optional[List[Dict[str, Any]]] + class AuthorizationResult(BaseSchema): allow: bool = False query: Optional[ProcessedQuery] debug: Optional[DebugInformation] - result: bool = False # fallback for older sdks (TODO: remove) + result: bool = False # fallback for older sdks (TODO: remove) diff --git a/horizon/local/api.py b/horizon/local/api.py index 7821b2d2..148f9e88 100644 --- a/horizon/local/api.py +++ b/horizon/local/api.py @@ -1,12 +1,13 @@ -from typing import Dict, Any, List, Optional +from typing import Any, Dict, List, Optional -from fastapi import APIRouter, status, HTTPException, Depends -from opal_client.policy_store import BasePolicyStoreClient, DEFAULT_POLICY_STORE_GETTER +from fastapi import APIRouter, Depends, HTTPException, status +from opal_client.policy_store import DEFAULT_POLICY_STORE_GETTER, BasePolicyStoreClient from horizon.authentication import enforce_pdp_token from horizon.local.schemas import Message, SyncedRole, SyncedUser -def init_local_cache_api_router(policy_store:BasePolicyStoreClient=None): + +def init_local_cache_api_router(policy_store: BasePolicyStoreClient = None): policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER() router = APIRouter(dependencies=[Depends(enforce_pdp_token)]) @@ -22,7 +23,7 @@ async def get_data_for_synced_user(user_id: str) -> Dict[str, Any]: if result is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"user with id '{user_id}' was not found in OPA cache! (not synced)" + detail=f"user with id '{user_id}' was not found in OPA cache! (not synced)", ) return result @@ -34,13 +35,14 @@ def permission_shortname(permission: Dict[str, Any]) -> Optional[str]: return None return f"{resource}:{action}" - @router.get( "/users/{user_id}", response_model=SyncedUser, responses={ - 404: error_message("User not found (i.e: not synced to Authorization service)"), - } + 404: error_message( + "User not found (i.e: not synced to Authorization service)" + ), + }, ) async def get_user(user_id: str): """ @@ -49,8 +51,8 @@ async def get_user(user_id: str): If user does not exist in OPA cache (i.e: not synced), returns 404. """ result = await get_data_for_synced_user(user_id) - roles=result.get("roles", []) - roles=[ + roles = result.get("roles", []) + roles = [ SyncedRole( id=r.get("id"), name=r.get("name"), @@ -72,7 +74,7 @@ async def get_user(user_id: str): response_model=List[SyncedUser], responses={ 404: error_message("OPA has no users stored in cache"), - } + }, ) async def list_users(): """ @@ -85,12 +87,12 @@ async def list_users(): if result is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"OPA has no users stored in cache! Did you synced users yet via the sdk or the cloud console?" + detail=f"OPA has no users stored in cache! Did you synced users yet via the sdk or the cloud console?", ) users = [] for user_id, user_data in iter(result.items()): - roles=user_data.get("roles", []) - roles=[ + roles = user_data.get("roles", []) + roles = [ SyncedRole( id=r.get("id"), name=r.get("name"), @@ -113,8 +115,10 @@ async def list_users(): "/users/{user_id}/roles", response_model=List[SyncedRole], responses={ - 404: error_message("User not found (i.e: not synced to Authorization service)"), - } + 404: error_message( + "User not found (i.e: not synced to Authorization service)" + ), + }, ) async def get_user_roles(user_id: str): """ @@ -126,9 +130,9 @@ async def get_user_roles(user_id: str): result = await get_data_for_synced_user(user_id) # will issue *another* opa request to list all roles, not just the roles for this user cached_roles: List[SyncedRole] = await list_roles() - role_data = {role.id: role for role in cached_roles } + role_data = {role.id: role for role in cached_roles} - raw_roles=result.get("roles", []) + raw_roles = result.get("roles", []) roles = [] for r in raw_roles: @@ -148,8 +152,10 @@ async def get_user_roles(user_id: str): "/users/{user_id}/tenants", response_model=List[str], responses={ - 404: error_message("User not found (i.e: not synced to Authorization service)"), - } + 404: error_message( + "User not found (i.e: not synced to Authorization service)" + ), + }, ) async def get_user_tenants(user_id: str): """ @@ -171,7 +177,7 @@ async def get_user_tenants(user_id: str): response_model=List[SyncedRole], responses={ 404: error_message("OPA has no roles stored in cache"), - } + }, ) async def list_roles(): """ @@ -182,18 +188,20 @@ async def list_roles(): if result is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"OPA has no roles stored in cache! Did you define roles yet via the sdk or the cloud console?" + detail=f"OPA has no roles stored in cache! Did you define roles yet via the sdk or the cloud console?", ) roles = [] for role_id, role_data in iter(result.items()): - permissions = [permission_shortname(p) for p in role_data.get("permissions", [])] + permissions = [ + permission_shortname(p) for p in role_data.get("permissions", []) + ] permissions = [p for p in permissions if p is not None] roles.append( SyncedRole( id=role_id, name=role_data.get("name"), metadata=role_data.get("metadata", {}), - permissions=permissions + permissions=permissions, ) ) return roles @@ -203,7 +211,7 @@ async def list_roles(): response_model=SyncedRole, responses={ 404: error_message("Role not found"), - } + }, ) async def get_role_by_id(role_id: str): """ @@ -220,15 +228,17 @@ async def get_role_by_id(role_id: str): if role_data is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"No such role in OPA cache!" + detail=f"No such role in OPA cache!", ) - permissions = [permission_shortname(p) for p in role_data.get("permissions", [])] + permissions = [ + permission_shortname(p) for p in role_data.get("permissions", []) + ] permissions = [p for p in permissions if p is not None] role = SyncedRole( id=role_id, name=role_data.get("name"), metadata=role_data.get("metadata", {}), - permissions=permissions + permissions=permissions, ) return role @@ -237,7 +247,7 @@ async def get_role_by_id(role_id: str): response_model=SyncedRole, responses={ 404: error_message("Role not found"), - } + }, ) async def get_role_by_name(role_name: str): """ @@ -254,22 +264,24 @@ async def get_role_by_name(role_name: str): if result is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, - detail=f"OPA has no roles stored in cache!" + detail=f"OPA has no roles stored in cache!", ) for role_id, role_data in iter(result.items()): name = role_data.get("name") if name is None or name != role_name: continue - permissions = [permission_shortname(p) for p in role_data.get("permissions", [])] + permissions = [ + permission_shortname(p) for p in role_data.get("permissions", []) + ] permissions = [p for p in permissions if p is not None] return SyncedRole( id=role_id, name=name, metadata=role_data.get("metadata", {}), - permissions=permissions + permissions=permissions, ) raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"No such role in OPA cache!" + status_code=status.HTTP_404_NOT_FOUND, detail=f"No such role in OPA cache!" ) - return router \ No newline at end of file + + return router diff --git a/horizon/local/schemas.py b/horizon/local/schemas.py index 1c124d81..dc720cde 100644 --- a/horizon/local/schemas.py +++ b/horizon/local/schemas.py @@ -1,4 +1,5 @@ -from typing import Dict, Any, Optional, List +from typing import Any, Dict, List, Optional + from pydantic import BaseModel @@ -6,9 +7,11 @@ class BaseSchema(BaseModel): class Config: orm_mode = True + class Message(BaseModel): detail: str + class SyncedRole(BaseSchema): id: str name: str @@ -22,4 +25,4 @@ class SyncedUser(BaseSchema): name: Optional[str] email: Optional[str] metadata: Optional[Dict[str, Any]] - roles: List[SyncedRole] \ No newline at end of file + roles: List[SyncedRole] diff --git a/horizon/main.py b/horizon/main.py index 5e404e5d..c8d44567 100644 --- a/horizon/main.py +++ b/horizon/main.py @@ -1,28 +1,28 @@ -import sys import logging - -from uuid import uuid4 +import sys from typing import List +from uuid import uuid4 -from fastapi import FastAPI, status, Depends +from fastapi import Depends, FastAPI, status from fastapi.responses import RedirectResponse from logzio.handler import LogzioHandler +from opal_client.client import OpalClient +from opal_client.config import OpaLogFormat, opal_client_config, opal_common_config from opal_client.opa.options import OpaServerOptions - -from opal_common.logger import logger, Formatter from opal_common.confi import Confi -from opal_client.client import OpalClient -from opal_client.config import OpaLogFormat, opal_common_config, opal_client_config +from opal_common.logger import Formatter, logger from horizon.authentication import enforce_pdp_token from horizon.config import sidecar_config -from horizon.enforcer.opa.config_maker import get_opa_authz_policy_file_path, get_opa_config_file_path -from horizon.proxy.api import router as proxy_router from horizon.enforcer.api import init_enforcer_api_router +from horizon.enforcer.opa.config_maker import ( + get_opa_authz_policy_file_path, + get_opa_config_file_path, +) from horizon.local.api import init_local_cache_api_router +from horizon.proxy.api import router as proxy_router from horizon.startup.remote_config import RemoteConfigFetcher - OPA_LOGGER_MODULE = "opal_client.opa.logger" @@ -38,6 +38,7 @@ def apply_config(overrides_dict: dict, config_object: Confi): else: logger.warning(f"Ignored non-existing config key: {prefixed_key}") + class PermitPDP: """ Permit.io PDP (Policy Decision Point) @@ -59,20 +60,26 @@ class PermitPDP: - local api (wrappers on top of opa cache) - enforcer api (implementation of is_allowed()) """ + def __init__(self): self._setup_temp_logger() # fetch and apply config override from cloud control plane remote_config = RemoteConfigFetcher().fetch_config() if not remote_config: - logger.warning("Could not fetch config from cloud control plane, reverting to local config!") + logger.warning( + "Could not fetch config from cloud control plane, reverting to local config!" + ) else: logger.info("Applying config overrides from cloud control plane...") apply_config(remote_config.opal_common or {}, opal_common_config) apply_config(remote_config.opal_client or {}, opal_client_config) apply_config(remote_config.pdp or {}, sidecar_config) - if sidecar_config.OPA_BEARER_TOKEN_REQUIRED or sidecar_config.OPA_DECISION_LOG_ENABLED: + if ( + sidecar_config.OPA_BEARER_TOKEN_REQUIRED + or sidecar_config.OPA_DECISION_LOG_ENABLED + ): # we need to pass to OPAL a custom inline OPA config to enable these features self._configure_inline_opa_config() @@ -118,7 +125,8 @@ def _configure_monitoring(self): """ patch fastapi to enable tracing and monitoring """ - from ddtrace import patch, config + from ddtrace import config, patch + # Datadog APM patch(fastapi=True) # Override service name @@ -129,8 +137,13 @@ def _configure_cloud_logging(self, remote_context: dict = {}): if not sidecar_config.CENTRAL_LOG_ENABLED: return - if not sidecar_config.CENTRAL_LOG_TOKEN or len(sidecar_config.CENTRAL_LOG_TOKEN) == 0: - logger.warning("Centralized log is enabled, but token is not valid. Disabling sink.") + if ( + not sidecar_config.CENTRAL_LOG_TOKEN + or len(sidecar_config.CENTRAL_LOG_TOKEN) == 0 + ): + logger.warning( + "Centralized log is enabled, but token is not valid. Disabling sink." + ) return logzio_handler = LogzioHandler( @@ -153,13 +166,13 @@ def _configure_cloud_logging(self, remote_context: dict = {}): serialize=True, level=logging.INFO, format=formatter.format, - colorize=False, # no colors - enqueue=True, # make sure logging to cloud is done asyncronously and thread-safe - catch=True, # if sink throws exceptions, swallow them as not critical + colorize=False, # no colors + enqueue=True, # make sure logging to cloud is done asyncronously and thread-safe + catch=True, # if sink throws exceptions, swallow them as not critical ) def _configure_inline_opa_config(self): - inline_opa_config={} + inline_opa_config = {} if sidecar_config.OPA_DECISION_LOG_ENABLED: # decision logs needs to be configured via the config file @@ -174,11 +187,13 @@ def _configure_inline_opa_config(self): # append the bearer token authz policy to inline OPA config auth_policy_file_path = get_opa_authz_policy_file_path(sidecar_config) - inline_opa_config.update({ - "authorization":"basic", - "authentication":"token", - "files":[auth_policy_file_path] - }) + inline_opa_config.update( + { + "authorization": "basic", + "authentication": "token", + "files": [auth_policy_file_path], + } + ) logger.debug(f"setting OPAL_INLINE_OPA_CONFIG={inline_opa_config}") @@ -195,9 +210,11 @@ def _configure_inline_opa_config(self): def _override_app_metadata(self, app: FastAPI): app.title = "Permit.io PDP" - app.description = "The PDP (Policy decision point) container wraps Open Policy Agent (OPA) with a higher-level API intended for fine grained " + \ - "application-level authorization. The PDP automatically handles pulling policy updates in real-time " + \ - "from a centrally managed cloud-service (api.permit.io)." + app.description = ( + "The PDP (Policy decision point) container wraps Open Policy Agent (OPA) with a higher-level API intended for fine grained " + + "application-level authorization. The PDP automatically handles pulling policy updates in real-time " + + "from a centrally managed cloud-service (api.permit.io)." + ) app.version = "0.2.0" app.openapi_tags = sidecar_config.OPENAPI_TAGS_METADATA return app @@ -211,19 +228,42 @@ def _configure_api_routes(self, app: FastAPI): local_router = init_local_cache_api_router(policy_store=self._opal.policy_store) # include the api routes - app.include_router(enforcer_router, tags=["Authorization API"], dependencies=[Depends(enforce_pdp_token)]) - app.include_router(local_router, prefix="/local", tags=["Local Queries"], dependencies=[Depends(enforce_pdp_token)]) - app.include_router(proxy_router, tags=["Cloud API Proxy"], dependencies=[Depends(enforce_pdp_token)]) + app.include_router( + enforcer_router, + tags=["Authorization API"], + dependencies=[Depends(enforce_pdp_token)], + ) + app.include_router( + local_router, + prefix="/local", + tags=["Local Queries"], + dependencies=[Depends(enforce_pdp_token)], + ) + app.include_router( + proxy_router, + tags=["Cloud API Proxy"], + dependencies=[Depends(enforce_pdp_token)], + ) # TODO: remove this when clients update sdk version (legacy routes) - @app.post("/update_policy", status_code=status.HTTP_200_OK, include_in_schema=False, dependencies=[Depends(enforce_pdp_token)]) + @app.post( + "/update_policy", + status_code=status.HTTP_200_OK, + include_in_schema=False, + dependencies=[Depends(enforce_pdp_token)], + ) async def legacy_trigger_policy_update(): - response = RedirectResponse(url='/policy-updater/trigger') + response = RedirectResponse(url="/policy-updater/trigger") return response - @app.post("/update_policy_data", status_code=status.HTTP_200_OK, include_in_schema=False, dependencies=[Depends(enforce_pdp_token)]) + @app.post( + "/update_policy_data", + status_code=status.HTTP_200_OK, + include_in_schema=False, + dependencies=[Depends(enforce_pdp_token)], + ) async def legacy_trigger_data_update(): - response = RedirectResponse(url='/data-updater/trigger') + response = RedirectResponse(url="/data-updater/trigger") return response @property diff --git a/horizon/proxy/api.py b/horizon/proxy/api.py index 133baaef..efe20435 100644 --- a/horizon/proxy/api.py +++ b/horizon/proxy/api.py @@ -1,16 +1,15 @@ import json import re -from typing import Dict, List, Any, Optional - -import aiohttp +from typing import Any, Dict, List, Optional from urllib.parse import urlparse -from fastapi import APIRouter, status, Request, HTTPException, Response +import aiohttp +from fastapi import APIRouter, HTTPException, Request, Response, status from fastapi.responses import JSONResponse +from opal_client.config import OpalClientConfig, opal_client_config from opal_client.utils import proxy_response from opal_common.logger import logger -from opal_client.config import OpalClientConfig, opal_client_config -from pydantic import parse_obj_as, BaseModel, Field +from pydantic import BaseModel, Field, parse_obj_as from horizon.config import sidecar_config @@ -36,9 +35,12 @@ class JSONPatchAction(BaseModel): """ Abstract base class for JSON patch actions (RFC 6902) """ + op: str = Field(..., description="patch action to perform") path: str = Field(..., description="target location in modified json") - value: Optional[Dict[str, Any]] = Field(None, description="json document, the operand of the action") + value: Optional[Dict[str, Any]] = Field( + None, description="json document, the operand of the action" + ) router = APIRouter() @@ -69,9 +71,7 @@ async def patch_handler(response: Response) -> Response: del response_json["patch"] del response.headers["Content-Length"] return JSONResponse( - response_json, - status_code=response.status_code, - headers=dict(response.headers) + response_json, status_code=response.status_code, headers=dict(response.headers) ) @@ -79,7 +79,7 @@ async def patch_handler(response: Response) -> Response: ("PUT", re.compile("users")), ("DELETE", re.compile("users\\/.+")), ("POST", re.compile("role_assignments")), - ("DELETE", re.compile("role_assignments")) + ("DELETE", re.compile("role_assignments")), } @@ -97,10 +97,12 @@ async def cloud_proxy(request: Request, path: str): if write_route: headers["X-Include-Patch"] = "true" - response = await proxy_request_to_cloud_service(request, - path, - cloud_service_url=sidecar_config.BACKEND_SERVICE_URL, - additional_headers=headers) + response = await proxy_request_to_cloud_service( + request, + path, + cloud_service_url=sidecar_config.BACKEND_SERVICE_URL, + additional_headers=headers, + ) if write_route: return await patch_handler(response) @@ -109,26 +111,38 @@ async def cloud_proxy(request: Request, path: str): # TODO: remove this once we migrate all clients -@router.api_route("/sdk/{path:path}", methods=ALL_METHODS, summary="Old Proxy Endpoint", include_in_schema=False) +@router.api_route( + "/sdk/{path:path}", + methods=ALL_METHODS, + summary="Old Proxy Endpoint", + include_in_schema=False, +) async def old_proxy(request: Request, path: str): - return await proxy_request_to_cloud_service(request, - path, - cloud_service_url=sidecar_config.BACKEND_LEGACY_URL, - additional_headers={}) + return await proxy_request_to_cloud_service( + request, + path, + cloud_service_url=sidecar_config.BACKEND_LEGACY_URL, + additional_headers={}, + ) -async def proxy_request_to_cloud_service(request: Request, path: str, cloud_service_url: str, additional_headers: Dict[str, str]) -> Response: +async def proxy_request_to_cloud_service( + request: Request, + path: str, + cloud_service_url: str, + additional_headers: Dict[str, str], +) -> Response: auth_header = request.headers.get("Authorization") if auth_header is None: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Must provide a bearer token!", - headers={"WWW-Authenticate": "Bearer"} + headers={"WWW-Authenticate": "Bearer"}, ) path = f"{cloud_service_url}/{path}" params = dict(request.query_params) or {} - original_headers = {k.lower(): v for k,v in iter(dict(request.headers).items())} + original_headers = {k.lower(): v for k, v in iter(dict(request.headers).items())} headers = additional_headers # copy only required header @@ -141,35 +155,47 @@ async def proxy_request_to_cloud_service(request: Request, path: str, cloud_serv headers["host"] = urlparse(cloud_service_url).netloc except Exception as e: # fallback - logger.error(f"could not urlparse cloud service url: {cloud_service_url}, exception: {e}") + logger.error( + f"could not urlparse cloud service url: {cloud_service_url}, exception: {e}" + ) logger.info(f"Proxying request: {request.method} {path}") async with aiohttp.ClientSession() as session: if request.method == HTTP_GET: - async with session.get(path, headers=headers, params=params) as backend_response: + async with session.get( + path, headers=headers, params=params + ) as backend_response: return await proxy_response(backend_response) if request.method == HTTP_DELETE: - async with session.delete(path, headers=headers, params=params) as backend_response: + async with session.delete( + path, headers=headers, params=params + ) as backend_response: return await proxy_response(backend_response) # these methods has data payload data = await request.body() if request.method == HTTP_POST: - async with session.post(path, headers=headers, data=data, params=params) as backend_response: + async with session.post( + path, headers=headers, data=data, params=params + ) as backend_response: return await proxy_response(backend_response) if request.method == HTTP_PUT: - async with session.put(path, headers=headers, data=data, params=params) as backend_response: + async with session.put( + path, headers=headers, data=data, params=params + ) as backend_response: return await proxy_response(backend_response) if request.method == HTTP_PATCH: - async with session.patch(path, headers=headers, data=data, params=params) as backend_response: + async with session.patch( + path, headers=headers, data=data, params=params + ) as backend_response: return await proxy_response(backend_response) raise HTTPException( status_code=status.HTTP_405_METHOD_NOT_ALLOWED, - detail="This method is not supported" + detail="This method is not supported", ) diff --git a/horizon/startup/remote_config.py b/horizon/startup/remote_config.py index ca606071..90f89655 100644 --- a/horizon/startup/remote_config.py +++ b/horizon/startup/remote_config.py @@ -1,9 +1,9 @@ -import requests from typing import Optional -from tenacity import retry, wait, stop, retry_if_not_exception_type -from pydantic import ValidationError +import requests from opal_common.logger import logger +from pydantic import ValidationError +from tenacity import retry, retry_if_not_exception_type, stop, wait from horizon.config import sidecar_config from horizon.startup.schemas import RemoteConfig @@ -49,11 +49,12 @@ class RemoteConfigFetcher: know the proper topic name. Otherwise opal client will receive updates for all organizations (which is not secure). """ + DEFAULT_RETRY_CONFIG = { - 'retry': retry_if_not_exception_type(InvalidPDPTokenException), - 'wait': wait.wait_random_exponential(max=10), - 'stop': stop.stop_after_attempt(10), - 'reraise': True, + "retry": retry_if_not_exception_type(InvalidPDPTokenException), + "wait": wait.wait_random_exponential(max=10), + "stop": stop.stop_after_attempt(10), + "reraise": True, } def __init__( @@ -61,7 +62,7 @@ def __init__( backend_url: str = sidecar_config.BACKEND_SERVICE_URL, sidecar_access_token: str = sidecar_config.API_KEY, remote_config_route: str = sidecar_config.REMOTE_CONFIG_ENDPOINT, - retry_config = None, + retry_config=None, ): """ inits the RemoteConfigFetcher. @@ -73,7 +74,9 @@ def __init__( """ self._url = f"{backend_url}/{remote_config_route}" self._token = sidecar_access_token - self._retry_config = retry_config if retry_config is not None else self.DEFAULT_RETRY_CONFIG + self._retry_config = ( + retry_config if retry_config is not None else self.DEFAULT_RETRY_CONFIG + ) def fetch_config(self) -> Optional[RemoteConfig]: """ @@ -104,13 +107,18 @@ def _fetch_config(self) -> RemoteConfig: try: sidecar_config = RemoteConfig(**response) - config_context = sidecar_config.dict(include={'context'}).get('context', {}) - logger.info(f"Received remote config with the following context: {config_context}") + config_context = sidecar_config.dict(include={"context"}).get( + "context", {} + ) + logger.info( + f"Received remote config with the following context: {config_context}" + ) except ValidationError as exc: - logger.error("Got invalid config contents: {exc}", exc=exc, response=response) + logger.error( + "Got invalid config contents: {exc}", exc=exc, response=response + ) raise return sidecar_config except requests.RequestException as exc: logger.error("Got exception: {exc}", exc=exc) raise - diff --git a/horizon/startup/schemas.py b/horizon/startup/schemas.py index ea21a9ee..9a0952db 100644 --- a/horizon/startup/schemas.py +++ b/horizon/startup/schemas.py @@ -1,7 +1,8 @@ from pydantic import BaseModel + class RemoteConfig(BaseModel): opal_common: dict = {} opal_client: dict = {} pdp: dict = {} - context: dict = {} \ No newline at end of file + context: dict = {} diff --git a/horizon/static/templates/authz.rego.template b/horizon/static/templates/authz.rego.template index e5c8c369..de434184 100644 --- a/horizon/static/templates/authz.rego.template +++ b/horizon/static/templates/authz.rego.template @@ -4,4 +4,4 @@ default allow = false # Reject requests by default. allow { # Allow request if... "{{ bearer_token }}" == input.identity # Identity is the secret root key. -} \ No newline at end of file +} diff --git a/horizon/static/templates/config.yaml.template b/horizon/static/templates/config.yaml.template index 7d436c94..1e468b7a 100644 --- a/horizon/static/templates/config.yaml.template +++ b/horizon/static/templates/config.yaml.template @@ -15,4 +15,4 @@ decision_logs: resource: {{ log_ingress_endpoint }} reporting: min_delay_seconds: {{ min_delay_seconds }} - max_delay_seconds: {{ max_delay_seconds }} \ No newline at end of file + max_delay_seconds: {{ max_delay_seconds }} diff --git a/scripts/gunicorn_conf.py b/scripts/gunicorn_conf.py index 9fa561f7..51ed5c24 100644 --- a/scripts/gunicorn_conf.py +++ b/scripts/gunicorn_conf.py @@ -22,4 +22,5 @@ def post_fork(server, worker): labels.update({"user": user}) import rook - rook.start(token=rookout_token, labels=labels) \ No newline at end of file + + rook.start(token=rookout_token, labels=labels) diff --git a/scripts/start.sh b/scripts/start.sh index d846d71d..798d2a6d 100644 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -9,4 +9,3 @@ if [[ -z "${PDP_ENABLE_MONITORING}" && "${PDP_ENABLE_MONITORING}" = "true" ]]; t else exec gunicorn -b 0.0.0.0:${UVICORN_PORT} -k uvicorn.workers.UvicornWorker --workers=${UVICORN_NUM_WORKERS} -c ${GUNICORN_CONF} ${UVICORN_ASGI_APP} fi - diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh index 3974640b..d990e0d3 100644 --- a/scripts/wait-for-it.sh +++ b/scripts/wait-for-it.sh @@ -179,4 +179,4 @@ if [[ $WAITFORIT_CLI != "" ]]; then exec "${WAITFORIT_CLI[@]}" else exit $WAITFORIT_RESULT -fi \ No newline at end of file +fi diff --git a/setup.py b/setup.py index 0a69b559..de7da89e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ import os import pathlib -from setuptools import setup, find_packages + +from setuptools import find_packages, setup + def get_requirements(env=""): if env: @@ -8,8 +10,11 @@ def get_requirements(env=""): with open("requirements{}.txt".format(env)) as fp: return [x.strip() for x in fp.read().split("\n") if not x.startswith("#")] + def get_data_files(root_directory: str): - all_files = [str(f) for f in pathlib.Path(f"{root_directory}/").glob("**/*") if f.is_file()] + all_files = [ + str(f) for f in pathlib.Path(f"{root_directory}/").glob("**/*") if f.is_file() + ] file_components = [(os.path.dirname(f), f) for f in all_files] grouped_files = {} for directory, fullpath in file_components: @@ -19,13 +24,14 @@ def get_data_files(root_directory: str): data_files.append((directory, fullpath)) return data_files + setup( - name='horizon', - version='0.2.0', + name="horizon", + version="0.2.0", packages=find_packages(), - python_requires='>=3.8', + python_requires=">=3.8", include_package_data=True, - data_files=get_data_files('horizon/static'), + data_files=get_data_files("horizon/static"), install_requires=get_requirements(), # dev_requires=get_requirements("dev"), -) \ No newline at end of file +)