fix: resolve unit test failure caused by differences in protobuf runtimes

WORKSPACE

@@ -2,9 +2,9 @@ workspace(name = "gapic_generator_python")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-_bazel_skylib_version = "0.9.0"
+_bazel_skylib_version = "1.4.0"
 
-_bazel_skylib_sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0"
+_bazel_skylib_sha256 = "f24ab666394232f834f74d19e2ff142b0af17466ea0c69a3f4c276ee75f6efce"
 
 http_archive(
     name = "bazel_skylib",
@@ -59,9 +59,7 @@ gapic_generator_python()
 
 gapic_generator_register_toolchains()
 
-load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 
-protobuf_deps()
 
 # TODO(https://github.com/googleapis/gapic-generator-python/issues/1781):
 # Remove this import once gRPC depends on a newer version.
@@ -81,16 +79,39 @@ http_archive(
     ],
 )
 
-#
-# Import grpc as a native bazel dependency. This avoids duplication and also
-# speeds up loading phase a lot (otherwise python_rules will be building grpcio
-# from sources in a single-core speed, which takes around 5 minutes on a regular
-# workstation)
-#
+_grpc_version = "1.55.1"
+
+_grpc_sha256 = "17c0685da231917a7b3be2671a7b13b550a85fdda5e475313264c5f51c4da3f8"
+
+http_archive(
+    name = "com_github_grpc_grpc",
+    sha256 = _grpc_sha256,
+    strip_prefix = "grpc-%s" % _grpc_version,
+    urls = ["https://github.com/grpc/grpc/archive/v%s.zip" % _grpc_version],
+)
+
+# Explicitly declaring Protobuf version, while Protobuf dependency is already
+# instantiated in grpc_deps().
+http_archive(
+    name = "com_google_protobuf",
+    sha256 = "0b0395d34e000f1229679e10d984ed7913078f3dd7f26cf0476467f5e65716f4",
+    strip_prefix = "protobuf-23.2",
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v23.2.tar.gz"],
+)
+
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
 
 grpc_deps()
 
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps", "PROTOBUF_MAVEN_ARTIFACTS")
+# This is actually already done within grpc_deps but calling this for Bazel convention.
+protobuf_deps()
+
+# gRPC enforces a specific version of Go toolchain which conflicts with our build.
+# All the relevant parts of grpc_extra_deps() are imported in this  WORKSPACE file
+# explicitly, that is why we do not call grpc_extra_deps() here and call
+# apple_rules_dependencies and apple_support_dependencies macros explicitly.
+
 load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
 
 apple_rules_dependencies()
@@ -106,3 +127,9 @@ switched_rules_by_language(
     gapic = True,
     grpc = True,
 )
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+python_register_toolchains(
+    name = "python39",
+    python_version = "3.9",
+)

gapic/templates/setup.py.j2

@@ -30,6 +30,7 @@ dependencies = [
     "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
     "proto-plus >= 1.22.0, <2.0.0dev",
     "proto-plus >= 1.22.2, <2.0.0dev; python_version>='3.11'",
+    {# Explicitly exclude protobuf versions mentioned in https://cloud.google.com/support/bulletins#GCP-2022-019 #}
     "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
     {% for package_tuple, package_info in pypi_packages.items() %}
     {# Quick check to make sure the package is different from this setup.py #}

gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2

@@ -887,50 +887,59 @@ def test_{{ method_name }}_rest(request_type):
     {% if not field.oneof or field.proto3_optional %}
     {# ignore oneof fields that might conflict with sample_request #}
     request_init["{{ field.name }}"] = {{ field.merged_mock_value(method.http_options[0].sample_request(method).get(field.name)) }}
-    # The version of a generated dependency at runtime may differ compared to the one at time of generation
-    # Delete any keys which are not present in the current runtime dependency
+    # The version of a generated dependency at test runtime may differ from the version used during generation
+    # Delete any fields which are not present in the current runtime dependency
     # See https://github.com/googleapis/gapic-generator-python/issues/1748
-    if hasattr({{ method.input.ident }}.meta.fields["{{ field.name }}"].message, "DESCRIPTOR"):
-        keys_to_delete = []
-
-        # Get all subfields for the message
-        subfield_names = [
-            (field.name, subfield.name)
-            for field in {{ method.input.ident }}.meta.fields["{{ field.name }}"].message.DESCRIPTOR.fields
-            if field.message_type
-            for subfield in field.message_type.fields
-        ]
 
-        # For each item in the sample request, create a list of sub fields which are not present at runtime
-        for key, value in request_init["{{ field.name }}"].items():
-            result = None
-            is_repeated = False
-            # For repeated fields
-            if isinstance(value, list) and len(value):
-                is_repeated = True
-                result = value[0]
-            # For fields where the type is another message
-            if isinstance(value, dict):
-                result = value
-
-            if result:
-                for nested_key in result.keys():
-                    if (key, nested_key) not in subfield_names:
-                        keys_to_delete.append(
-                            {"key": key, "nested_key": nested_key, "is_repeated": is_repeated}
-                        )
-
-        # Remove fields from the sample request which are not present in the runtime version of the dependency
-        for key_to_delete in keys_to_delete:
-            if key_to_delete.get("nested_key"):
-                if key_to_delete.get("is_repeated"):
-                    del request_init["{{ field.name }}"][key_to_delete.get("key")][0][
-                        key_to_delete.get("nested_key")
-                    ]
-                else:
-                    del request_init["{{ field.name }}"][key_to_delete.get("key")][
-                        key_to_delete.get("nested_key")
+    # Determine if the message type is proto-plus or protobuf
+    is_message_proto_plus_type = not hasattr({{ method.input.ident }}.meta.fields["{{ field.name }}"].message, "DESCRIPTOR")
+
+    if is_message_proto_plus_type:
+        message_fields = {{ method.input.ident }}.meta.fields["{{ field.name }}"].message.meta.fields
+    else:
+        message_fields = {{ method.input.ident }}.meta.fields["{{ field.name }}"].message.DESCRIPTOR.fields
+
+    subfields_not_in_runtime = []
+
+    # Get all subfields for the message
+    nested_fields = [
+        (field.name, subfield.name)
+        for field in message_fields
+        if hasattr(field, "message_type") and field.message_type
+        for subfield in field.message_type.fields
+    ]
+
+    # For each item in the sample request, create a list of sub fields which are not present at runtime
+    for field, value in request_init["{{ field.name }}"].items():
+        result = None
+        is_repeated = False
+        # For repeated fields
+        if isinstance(value, list) and len(value):
+            is_repeated = True
+            result = value[0]
+        # For fields where the type is another message
+        if isinstance(value, dict):
+            result = value
+
+        if result and hasattr(result, "keys"):
+            for subfield in result.keys():
+                if (field, subfield) not in nested_fields:
+                    subfields_not_in_runtime.append(
+                        {"field": field, "subfield": subfield, "is_repeated": is_repeated}
+                    )
+
+    # Remove fields from the sample request which are not present in the runtime version of the dependency
+    for subfield_to_delete in subfields_not_in_runtime:
+        if subfield_to_delete.get("subfield"):
+            if subfield_to_delete.get("is_repeated"):
+                for i in range(0, len(request_init["{{ field.name }}"][subfield_to_delete.get("field")])):
+                    del request_init["{{ field.name }}"][subfield_to_delete.get("field")][i][
+                        subfield_to_delete.get("subfield")
                     ]
+            else:
+                del request_init["{{ field.name }}"][subfield_to_delete.get("field")][
+                    subfield_to_delete.get("subfield")
+                ]
     {% endif %}
     {% endfor %}
     request = request_type(**request_init)

setup.py

@@ -25,8 +25,8 @@
 version = "1.11.5"
 release_status = "Development Status :: 5 - Production/Stable"
 dependencies = [
-    # Esnure that the lower bounds of these dependencies match what we have in the
-    # templated setup.py.j2 here https://github.com/googleapis/gapic-generator-python/blob/main/gapic/templates/setup.py.j2
+    # Ensure that the lower bounds of these dependencies match what we have in the
+    # templated setup.py.j2: https://github.com/googleapis/gapic-generator-python/blob/main/gapic/templates/setup.py.j2
     "click >= 6.7",
     "google-api-core[grpc] >= 1.34.0, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*",
     "googleapis-common-protos >= 1.55.0",

tests/fragments/google/README.rst

@@ -0,0 +1,3 @@
+The protos in this folder were copied directly from `googleapis/googleapis`_ and are needed for the purposes of running fragment tests.
+
+.. _googleapis/googleapis: https://github.com/googleapis/googleapis/tree/master/google