Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will revert this change once #18352 is merged to main and released

persist-credentials: false
- name: Setup Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
Expand All @@ -50,11 +47,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
Expand Down
55 changes: 54 additions & 1 deletion .github/workflows/gapic-generator-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ concurrency:
cancel-in-progress: true

env:
SHOWCASE_VERSION: 0.35.0
SHOWCASE_VERSION: 0.44.0
PROTOC_VERSION: 33.5
LATEST_STABLE_PYTHON: 3.14
OLDEST_PYTHON: '3.10'
PRERELEASE_PYTHON: 3.15
ALL_PYTHON: "['3.10', '3.11', '3.12', '3.13', '3.14', '3.15']"
TRIMMED_PYTHON: "['3.10', '3.14', '3.15']"
Expand Down Expand Up @@ -50,6 +51,7 @@ jobs:
outputs:
all_python: ${{ env.ALL_PYTHON }}
latest_stable_python: ${{ env.LATEST_STABLE_PYTHON }}
oldest_python: ${{ env.OLDEST_PYTHON }}
prerelease_python: ${{ env.PRERELEASE_PYTHON }}
trimmed_python: ${{ env.TRIMMED_PYTHON }}
steps:
Expand Down Expand Up @@ -94,6 +96,57 @@ jobs:
cd packages/gapic-generator
nox -s showcase_unit${{ matrix.variant }}-${MATRIX_PYTHON}

showcase:
needs: python_config
strategy:
# Run showcase tests on the lowest and highest supported runtimes
matrix:
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121) Remove `showcase_w_rest_async` target when async rest is GA.
python: ["${{ needs.python_config.outputs.oldest_python }}", "${{ needs.python_config.outputs.latest_stable_python }}"]
target: [showcase, showcase_w_rest_async]
logging_scope: ["", "google"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- name: Set up Python "${{ matrix.python }}"
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "${{ matrix.python }}"
cache: 'pip'
- name: Install system dependencies.
run: |
sudo apt-get update
sudo apt-get install -y curl pandoc unzip
- name: Install showcase
run: |
sudo mkdir -p /usr/src/showcase
sudo chown -R ${USER} /usr/src/
curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
cd /usr/src/showcase/
tar -xf showcase-*
./gapic-showcase run &
cd -
- name: Install nox.
run: python -m pip install nox
- name: Install protoc
run: |
sudo mkdir -p /usr/src/protoc/
sudo chown -R ${USER} /usr/src/
curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc-${PROTOC_VERSION}.zip
cd /usr/src/protoc/
unzip protoc-${PROTOC_VERSION}.zip
sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
- name: Run showcase tests.
env:
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2286): Construct nox sessions with logging enabled.
GOOGLE_SDK_PYTHON_LOGGING_SCOPE: ${{ matrix.logging_scope }}
MATRIX_PYTHON: ${{ matrix.python }}
run: |
cd packages/gapic-generator
nox -s ${{ matrix.target }}-${MATRIX_PYTHON}

showcase-mypy:
needs: python_config
runs-on: ubuntu-latest
Expand Down
10 changes: 2 additions & 8 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0
persist-credentials: false
- name: Check for unit_test:all_packages label
id: check-label
Expand Down Expand Up @@ -133,11 +130,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
Expand Down
15 changes: 3 additions & 12 deletions .github/workflows/unittest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -113,11 +113,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
Expand Down Expand Up @@ -187,11 +184,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
Expand Down Expand Up @@ -247,11 +241,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
# Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
# See https://github.com/googleapis/google-cloud-python/issues/12013
# and https://github.com/actions/checkout#checkout-head.
with:
fetch-depth: 2
fetch-depth: 0
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
Expand Down
32 changes: 31 additions & 1 deletion packages/gapic-generator/gapic/schema/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1345,7 +1345,8 @@ def _load_children(
wrapped = loader(
child, address=address, path=path + (i,), resources=resources
)
answer[wrapped.name] = wrapped
if wrapped is not None:
answer[wrapped.name] = wrapped
return answer

def _get_oneofs(
Expand Down Expand Up @@ -1633,6 +1634,9 @@ def _get_methods(
# Iterate over the methods and collect them into a dictionary.
answer: Dict[str, wrappers.Method] = collections.OrderedDict()
for i, meth_pb in enumerate(methods):
if self._is_media_upload_proto(meth_pb) and not self.opts.resumable_upload_prefix:
continue
Comment on lines +1637 to +1638

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Skipping the generation of media upload methods entirely when resumable_upload_prefix is not configured can introduce breaking changes for downstream users who previously relied on these methods being generated as standard RPCs. Consider allowing them to be generated as standard (non-resumable) methods instead of omitting them completely.

References
  1. Do not replace historical graceful fallback behaviors with exceptions or omit them if doing so would introduce breaking changes for downstream users and violate backwards compatibility.


retry, timeout = self._get_retry_and_timeout(service_address, meth_pb)

# Create the method wrapper object.
Expand All @@ -1651,11 +1655,37 @@ def _get_methods(
output=self.api_messages[meth_pb.output_type.lstrip(".")],
retry=retry,
timeout=timeout,
resumable_upload_prefix=self.opts.resumable_upload_prefix,
)

# Done; return the answer.
return answer

def _is_media_upload_proto(
self, meth_pb: descriptor_pb2.MethodDescriptorProto
) -> bool:
try:
if meth_pb.options:
http = meth_pb.options.Extensions[annotations_pb2.http]
if getattr(http, "media_upload", None) and getattr(
http.media_upload, "enabled", False
):
return True
for binding in getattr(http, "additional_bindings", ()):
if getattr(binding, "media_upload", None) and getattr(
binding.media_upload, "enabled", False
):
return True
except Exception:
pass

# TODO(cl/964122389): TEMPORARY - Remove this hardcoded fallback once
# the media_upload annotation is published in cl/964122389 and added to gapic-showcase proto.
if meth_pb.name == "UploadMedia":
return True

return False

def _load_message(
self,
message_pb: descriptor_pb2.DescriptorProto,
Expand Down
32 changes: 32 additions & 0 deletions packages/gapic-generator/gapic/schema/wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1499,6 +1499,7 @@ class Method:
meta: metadata.Metadata = dataclasses.field(
default_factory=metadata.Metadata,
)
resumable_upload_prefix: str = "resumable/upload"

def __getattr__(self, name):
return getattr(self.method_pb, name)
Expand Down Expand Up @@ -1728,6 +1729,32 @@ def http_opt(self) -> Optional[Dict[str, str]]:
# TODO(yon-mg): enums for http verbs?
return answer

@property
def is_resumable_upload(self) -> bool:
"""Return True if this method is a resumable upload method."""
if not self.resumable_upload_prefix:
return False

try:
if hasattr(self, "options") and self.options:
http = self.options.Extensions[annotations_pb2.http]
if getattr(http, "media_upload", None) and getattr(http.media_upload, "enabled", False):
return True
for binding in getattr(http, "additional_bindings", ()):
if getattr(binding, "media_upload", None) and getattr(binding.media_upload, "enabled", False):
return True
except Exception:
pass

# TODO(cl/964122389): TEMPORARY - Remove this hardcoded fallback once
# the media_upload annotation is published in cl/964122389 and added to gapic-showcase proto.
pb_name = getattr(self.method_pb, "name", "")
method_name = getattr(self, "name", "")
if pb_name == "UploadMedia" or method_name == "upload_media":
return True

return False

@property
def path_params(self) -> Sequence[str]:
"""Return the path parameters found in the http annotation path template"""
Expand Down Expand Up @@ -2208,6 +2235,11 @@ def has_pagers(self) -> bool:
"""Return whether the service has paged methods."""
return any(m.paged_result_field for m in self.methods.values())

@property
def has_resumable_upload_methods(self) -> bool:
"""Return whether the service has resumable upload methods."""
return any(m.is_resumable_upload for m in self.methods.values())

@property
def host(self) -> str:
"""Return the hostname for this service, if specified.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@ except ImportError: # pragma: NO COVER
{% endmacro %}

{% macro create_metadata(method) %}
{% if method.is_resumable_upload %}
metadata = () if metadata is None else metadata
resumable_metadata = {
"x-goog-upload-protocol": "resumable",
"x-goog-upload-command": "start",
}
existing_keys = {k.lower() for k, _ in metadata}
metadata = tuple(metadata) + tuple(
(k, v) for k, v in resumable_metadata.items() if k not in existing_keys
)
{% endif %}
{% if method.explicit_routing %}
header_params: dict[str, str] = {}
{% if not method.client_streaming %}
Expand Down Expand Up @@ -132,13 +143,13 @@ from google.longrunning import operations_pb2 # type: ignore
{% endif %}{# import_ns.has_operations_mixin #}
{% endmacro %}

{% macro http_options_method(rules) %}
{% macro http_options_method(rules, is_resumable_upload=False, resumable_upload_prefix="resumable/upload") %}
@staticmethod
def _get_http_options():
http_options: List[Dict[str, str]] = [
{%- for rule in rules %}{
'method': '{{ rule.method }}',
'uri': '{{ rule.uri }}',
'uri': '{% if is_resumable_upload %}/{{ resumable_upload_prefix }}{% endif %}{{ rule.uri }}',
{% if rule.body %}
'body': '{{ rule.body }}',
{% endif %}{# rule.body #}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,9 @@ class {{ service.name }}Transport(abc.ABC):

# Save the credentials.
self._credentials = credentials
{% if service.has_resumable_upload_methods %}
self._client_info = client_info
{% endif %}

# Save the hostname. Default to port 443 (HTTPS) if none is specified.
if ':' not in host:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import pickle
import warnings
from typing import Callable, Dict, Optional, Sequence, Tuple, Union

{% if service.has_resumable_upload_methods %}
from google.api_core import exceptions as core_exceptions
{% endif %}
from google.api_core import grpc_helpers
{% if service.has_lro %}
from google.api_core import operations_v1
Expand Down Expand Up @@ -49,6 +52,9 @@ from google.longrunning import operations_pb2 # type: ignore
{% endif %}
{% endfilter %}
from .base import {{ service.name }}Transport, DEFAULT_CLIENT_INFO
{% if service.has_resumable_upload_methods %}
from .rest import {{ service.name }}RestTransport
{% endif %}

try:
from google.api_core import client_logging # type: ignore
Expand Down Expand Up @@ -353,11 +359,28 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
# gRPC handles serialization and deserialization, so we just need
# to pass in the functions for each.
if '{{ method.transport_safe_name|snake_case }}' not in self._stubs:
{% if method.is_resumable_upload %}
if not self._credentials:
class _ErrorStub:
def __call__(self, *args, **kwargs):
raise core_exceptions.GoogleAPICallError(
"Resumable upload methods operate over REST and cannot be invoked when the transport is initialized with a pre-constructed gRPC channel. Please supply credentials directly instead of a gRPC channel to use resumable upload functionality."
)
self._stubs['{{ method.transport_safe_name|snake_case }}'] = _ErrorStub()
else:
rest_transport = {{ service.name }}RestTransport(
host=self._host,
credentials=self._credentials,
client_info=self._client_info,
)
self._stubs['{{ method.transport_safe_name|snake_case }}'] = rest_transport.{{ method.transport_safe_name|snake_case }}
{% else %}
self._stubs['{{ method.transport_safe_name|snake_case }}'] = self._logged_channel.{{ method.grpc_stub_type }}(
'/{{ '.'.join(method.meta.address.package) }}.{{ service.name }}/{{ method.name }}',
request_serializer={{ method.input.ident }}.{% if method.input.ident.python_import.module.endswith('_pb2') %}SerializeToString{% else %}serialize{% endif %},
response_deserializer={{ method.output.ident }}.{% if method.output.ident.python_import.module.endswith('_pb2') %}FromString{% else %}deserialize{% endif %},
)
{% endif %}
return self._stubs['{{ method.transport_safe_name|snake_case }}']
{% endfor %}

Expand Down
Loading
Loading