Skip to content
This repository was archived by the owner on Jun 8, 2026. It is now read-only.
This repository was archived by the owner on Jun 8, 2026. It is now read-only.

tests.system.test_table_api: test_table_exists failed #512

Closed

Description

@flaky-bot

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.


commit: bdd5f8b
buildURL: Build Status, Sponge
status: failed

Test output
target = functools.partial(>)
predicate = .if_exception_type_predicate at 0x7f6d8182d3a0>
sleep_generator = 
deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
          return target()

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:


self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>
retry = <google.api_core.retry.Retry object at 0x7f6d81831430>

def _done_or_raise(self, retry=DEFAULT_RETRY):
    """Check if the future is done and raise if it's not."""
    kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

    if not self.done(**kwargs):
      raise _OperationNotComplete()

E google.api_core.future.polling._OperationNotComplete

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete

The above exception was the direct cause of the following exception:

self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f6d81831430>

def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
      retry_(self._done_or_raise)(**kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:


args = (), kwargs = {}
target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>)
sleep_generator = <generator object exponential_sleep_generator at 0x7f6d812e1510>

@general_helpers.wraps(func)
def retry_wrapped_func(*args, **kwargs):
    """A wrapper that calls target function with retry."""
    target = functools.partial(func, *args, **kwargs)
    sleep_generator = exponential_sleep_generator(
        self._initial, self._maximum, multiplier=self._multiplier
    )
  return retry_target(
        target,
        self._predicate,
        sleep_generator,
        self._deadline,
        on_error=on_error,
    )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:


target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>)
predicate = <function if_exception_type..if_exception_type_predicate at 0x7f6d8182d3a0>
sleep_generator = <generator object exponential_sleep_generator at 0x7f6d812e1510>
deadline = 60, on_error = None

def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
            return target()

        # pylint: disable=broad-except
        # This function explicitly must deal with broad exceptions.
        except Exception as exc:
            if not predicate(exc):
                raise
            last_exc = exc
            if on_error is not None:
                on_error(exc)

        now = datetime_helpers.utcnow()

        if deadline_datetime is not None:
            if deadline_datetime <= now:
              six.raise_from(
                    exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
                    ),
                    last_exc,
                )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:


value = None, from_value = _OperationNotComplete()

???
E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>), last exception:

:3: RetryError

During handling of the above exception, another exception occurred:

spanner_client = <google.cloud.spanner_v1.client.Client object at 0x7f6d811926d0>
operation_timeout = 60, shared_instance_id = 'google-cloud-1628986419564'
instance_config = name: "projects/precise-truck-742/instanceConfigs/regional-us-central1"
display_name: "us-central1"
replicas {
locat...
replicas {
location: "us-central1"
type_: READ_WRITE
}
replicas {
location: "us-central1"
type_: READ_WRITE
}

existing_instances = [name: "projects/precise-truck-742/instances/gcp-streaming-systests"
config: "projects/precise-truck-742/instanceConfi...ated"
value: "1628986209"
}
labels {
key: "python-spanner-systests"
value: "true"
}
processing_units: 1000
, ...]

@pytest.fixture(scope="session")
def shared_instance(
    spanner_client,
    operation_timeout,
    shared_instance_id,
    instance_config,
    existing_instances,  # evalutate before creating one
):
    _helpers.cleanup_old_instances(spanner_client)

    if _helpers.CREATE_INSTANCE:
        create_time = str(int(time.time()))
        labels = {"python-spanner-systests": "true", "created": create_time}

        instance = spanner_client.instance(
            shared_instance_id, instance_config.name, labels=labels
        )
        created_op = _helpers.retry_429_503(instance.create)()
      created_op.result(operation_timeout)  # block until completion

tests/system/conftest.py:119:


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
self._blocking_poll(timeout=timeout, **kwargs)


self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>
timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f6d81831430>

def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
        retry_(self._done_or_raise)(**kwargs)
    except exceptions.RetryError:
      raise concurrent.futures.TimeoutError(
            "Operation did not complete within the designated " "timeout."
        )

E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

Activity

  1. added
    flakybot: issueAn issue filed by the Flaky Bot. Should not be added manually.
    priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.
    type: bugError or flaw in code with unintended results or allowing sub-optimal usage patterns.
    on Aug 15, 2021
  2. flaky-bot commented on Aug 15, 2021

    @flaky-bot
    Author

    Test passed for commit 372b40d (Build Status, Sponge)! Closing this issue.

  3. added
    flakybot: flakyTells the Flaky Bot not to close or comment on this issue.
    on Aug 16, 2021
  4. flaky-bot commented on Aug 16, 2021

    @flaky-bot
    Author

    Looks like this issue is flaky. 😟

    I'm going to leave this open and stop commenting.

    A human should fix and close this.


    commit: 05a2c37
    buildURL: Build Status, Sponge
    status: failed

    Test output
    target = functools.partial(>)
    predicate = .if_exception_type_predicate at 0x7f5082b113a0>
    sleep_generator = 
    deadline = 60, on_error = None
    
    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
    
              return target()
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:


    self = <google.api_core.operation.Operation object at 0x7f50824b8760>
    retry = <google.api_core.retry.Retry object at 0x7f5082b109d0>

    def _done_or_raise(self, retry=DEFAULT_RETRY):
        """Check if the future is done and raise if it's not."""
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
        if not self.done(**kwargs):
    
          raise _OperationNotComplete()
    

    E google.api_core.future.polling._OperationNotComplete

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete

    The above exception was the direct cause of the following exception:

    self = <google.api_core.operation.Operation object at 0x7f50824b8760>
    timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f5082b109d0>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
          retry_(self._done_or_raise)(**kwargs)
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:


    args = (), kwargs = {}
    target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f50824b8760>>)
    sleep_generator = <generator object exponential_sleep_generator at 0x7f5082587580>

    @general_helpers.wraps(func)
    def retry_wrapped_func(*args, **kwargs):
        """A wrapper that calls target function with retry."""
        target = functools.partial(func, *args, **kwargs)
        sleep_generator = exponential_sleep_generator(
            self._initial, self._maximum, multiplier=self._multiplier
        )
    
      return retry_target(
    
            target,
            self._predicate,
            sleep_generator,
            self._deadline,
            on_error=on_error,
        )
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:


    target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f50824b8760>>)
    predicate = <function if_exception_type..if_exception_type_predicate at 0x7f5082b113a0>
    sleep_generator = <generator object exponential_sleep_generator at 0x7f5082587580>
    deadline = 60, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
                return target()
    
            # pylint: disable=broad-except
            # This function explicitly must deal with broad exceptions.
            except Exception as exc:
                if not predicate(exc):
                    raise
                last_exc = exc
                if on_error is not None:
                    on_error(exc)
    
            now = datetime_helpers.utcnow()
    
            if deadline_datetime is not None:
                if deadline_datetime <= now:
    
                  six.raise_from(
    
                        exceptions.RetryError(
                            "Deadline of {:.1f}s exceeded while calling {}".format(
                                deadline, target
                            ),
                            last_exc,
                        ),
                        last_exc,
                    )
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:


    value = None, from_value = _OperationNotComplete()

    ???
    E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f50824b8760>>), last exception:

    :3: RetryError

    During handling of the above exception, another exception occurred:

    spanner_client = <google.cloud.spanner_v1.client.Client object at 0x7f50824a5850>
    operation_timeout = 60, shared_instance_id = 'google-cloud-1629077125273'
    instance_config = name: "projects/precise-truck-742/instanceConfigs/regional-us-central1"
    display_name: "us-central1"
    replicas {
    locat...
    replicas {
    location: "us-central1"
    type_: READ_WRITE
    }
    replicas {
    location: "us-central1"
    type_: READ_WRITE
    }

    existing_instances = [name: "projects/precise-truck-742/instances/gcp-streaming-systests"
    config: "projects/precise-truck-742/instanceConfi...: "created"
    value: "1627588424"
    }
    labels {
    key: "sample_name"
    value: "autocommit"
    }
    processing_units: 1000
    , ...]

    @pytest.fixture(scope="session")
    def shared_instance(
        spanner_client,
        operation_timeout,
        shared_instance_id,
        instance_config,
        existing_instances,  # evalutate before creating one
    ):
        _helpers.cleanup_old_instances(spanner_client)
    
        if _helpers.CREATE_INSTANCE:
            create_time = str(int(time.time()))
            labels = {"python-spanner-systests": "true", "created": create_time}
    
            instance = spanner_client.instance(
                shared_instance_id, instance_config.name, labels=labels
            )
            created_op = _helpers.retry_429_503(instance.create)()
    
          created_op.result(operation_timeout)  # block until completion
    

    tests/system/conftest.py:119:


    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
    self._blocking_poll(timeout=timeout, **kwargs)


    self = <google.api_core.operation.Operation object at 0x7f50824b8760>
    timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f5082b109d0>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
            retry_(self._done_or_raise)(**kwargs)
        except exceptions.RetryError:
    
          raise concurrent.futures.TimeoutError(
    
                "Operation did not complete within the designated " "timeout."
            )
    

    E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

  5. flaky-bot commented on Aug 16, 2021

    @flaky-bot
    Author

    Oops! Looks like this issue is still flaky. It failed again. 😬

    I reopened the issue, but a human will need to close it again.


    commit: 492ad09
    buildURL: Build Status, Sponge
    status: failed

    Test output
    target = functools.partial(>)
    predicate = .if_exception_type_predicate at 0x7fdd86b883a0>
    sleep_generator = 
    deadline = 60, on_error = None
    
    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
    
              return target()
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:


    self = <google.api_core.operation.Operation object at 0x7fdd864f9520>
    retry = <google.api_core.retry.Retry object at 0x7fdd86b87b80>

    def _done_or_raise(self, retry=DEFAULT_RETRY):
        """Check if the future is done and raise if it's not."""
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
        if not self.done(**kwargs):
    
          raise _OperationNotComplete()
    

    E google.api_core.future.polling._OperationNotComplete

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete

    The above exception was the direct cause of the following exception:

    self = <google.api_core.operation.Operation object at 0x7fdd864f9520>
    timeout = 60, retry = <google.api_core.retry.Retry object at 0x7fdd86b87b80>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
    
          retry_(self._done_or_raise)(**kwargs)
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:


    args = (), kwargs = {}
    target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fdd864f9520>>)
    sleep_generator = <generator object exponential_sleep_generator at 0x7fdd866453c0>

    @general_helpers.wraps(func)
    def retry_wrapped_func(*args, **kwargs):
        """A wrapper that calls target function with retry."""
        target = functools.partial(func, *args, **kwargs)
        sleep_generator = exponential_sleep_generator(
            self._initial, self._maximum, multiplier=self._multiplier
        )
    
      return retry_target(
    
            target,
            self._predicate,
            sleep_generator,
            self._deadline,
            on_error=on_error,
        )
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:


    target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fdd864f9520>>)
    predicate = <function if_exception_type..if_exception_type_predicate at 0x7fdd86b883a0>
    sleep_generator = <generator object exponential_sleep_generator at 0x7fdd866453c0>
    deadline = 60, on_error = None

    def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
        """Call a function and retry if it fails.
    
        This is the lowest-level retry helper. Generally, you'll use the
        higher-level retry helper :class:`Retry`.
    
        Args:
            target(Callable): The function to call and retry. This must be a
                nullary function - apply arguments with `functools.partial`.
            predicate (Callable[Exception]): A callable used to determine if an
                exception raised by the target should be considered retryable.
                It should return True to retry or False otherwise.
            sleep_generator (Iterable[float]): An infinite iterator that determines
                how long to sleep between retries.
            deadline (float): How long to keep retrying the target. The last sleep
                period is shortened as necessary, so that the last retry runs at
                ``deadline`` (and not considerably beyond it).
            on_error (Callable[Exception]): A function to call while processing a
                retryable exception.  Any error raised by this function will *not*
                be caught.
    
        Returns:
            Any: the return value of the target function.
    
        Raises:
            google.api_core.RetryError: If the deadline is exceeded while retrying.
            ValueError: If the sleep generator stops yielding values.
            Exception: If the target raises a method that isn't retryable.
        """
        if deadline is not None:
            deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
                seconds=deadline
            )
        else:
            deadline_datetime = None
    
        last_exc = None
    
        for sleep in sleep_generator:
            try:
                return target()
    
            # pylint: disable=broad-except
            # This function explicitly must deal with broad exceptions.
            except Exception as exc:
                if not predicate(exc):
                    raise
                last_exc = exc
                if on_error is not None:
                    on_error(exc)
    
            now = datetime_helpers.utcnow()
    
            if deadline_datetime is not None:
                if deadline_datetime <= now:
    
                  six.raise_from(
    
                        exceptions.RetryError(
                            "Deadline of {:.1f}s exceeded while calling {}".format(
                                deadline, target
                            ),
                            last_exc,
                        ),
                        last_exc,
                    )
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:


    value = None, from_value = _OperationNotComplete()

    ???
    E google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fdd864f9520>>), last exception:

    :3: RetryError

    During handling of the above exception, another exception occurred:

    spanner_client = <google.cloud.spanner_v1.client.Client object at 0x7fdd864e6be0>
    operation_timeout = 60, shared_instance_id = 'google-cloud-1629113513103'
    instance_config = name: "projects/precise-truck-742/instanceConfigs/regional-us-central1"
    display_name: "us-central1"
    replicas {
    locat...
    replicas {
    location: "us-central1"
    type_: READ_WRITE
    }
    replicas {
    location: "us-central1"
    type_: READ_WRITE
    }

    existing_instances = [name: "projects/precise-truck-742/instances/gcp-streaming-systests"
    config: "projects/precise-truck-742/instanceConfi...: "created"
    value: "1627588424"
    }
    labels {
    key: "sample_name"
    value: "autocommit"
    }
    processing_units: 1000
    , ...]

    @pytest.fixture(scope="session")
    def shared_instance(
        spanner_client,
        operation_timeout,
        shared_instance_id,
        instance_config,
        existing_instances,  # evalutate before creating one
    ):
        _helpers.cleanup_old_instances(spanner_client)
    
        if _helpers.CREATE_INSTANCE:
            create_time = str(int(time.time()))
            labels = {"python-spanner-systests": "true", "created": create_time}
    
            instance = spanner_client.instance(
                shared_instance_id, instance_config.name, labels=labels
            )
            created_op = _helpers.retry_429_503(instance.create)()
    
          created_op.result(operation_timeout)  # block until completion
    

    tests/system/conftest.py:119:


    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result
    self._blocking_poll(timeout=timeout, **kwargs)


    self = <google.api_core.operation.Operation object at 0x7fdd864f9520>
    timeout = 60, retry = <google.api_core.retry.Retry object at 0x7fdd86b87b80>

    def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
        """Poll and wait for the Future to be resolved.
    
        Args:
            timeout (int):
                How long (in seconds) to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return
    
        retry_ = self._retry.with_deadline(timeout)
    
        try:
            kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
            retry_(self._done_or_raise)(**kwargs)
        except exceptions.RetryError:
    
          raise concurrent.futures.TimeoutError(
    
                "Operation did not complete within the designated " "timeout."
            )
    

    E concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

  6. larkee commented on Aug 18, 2021

    @larkee
    Contributor

    Closed by #525

  7. flaky-bot commented on Aug 24, 2021

    @flaky-bot
    Author

    Oops! Looks like this issue is still flaky. It failed again. 😬

    I reopened the issue, but a human will need to close it again.


    commit: 098a786
    buildURL: Build Status, Sponge
    status: failed

    Test output
    self = 
    
    def get(self):
        """Check a session out from the pool.
    
        :rtype: :class:`~google.cloud.spanner_v1.session.Session`
        :returns: an existing session from the pool, or a newly-created
                  session.
        """
        try:
    
          session = self._sessions.get_nowait()
    

    google/cloud/spanner_v1/pool.py:272:


    self = <queue.LifoQueue object at 0x7f24d9075c10>

    def get_nowait(self):
        '''Remove and return an item from the queue without blocking.
    
        Only get an item if one is immediately available. Otherwise
        raise the Empty exception.
        '''
    
      return self.get(block=False)
    

    /usr/local/lib/python3.8/queue.py:198:


    self = <queue.LifoQueue object at 0x7f24d9075c10>, block = False, timeout = None

    def get(self, block=True, timeout=None):
        '''Remove and return an item from the queue.
    
        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until an item is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Empty exception if no item was available within that time.
        Otherwise ('block' is false), return an item if one is immediately
        available, else raise the Empty exception ('timeout' is ignored
        in that case).
        '''
        with self.not_empty:
            if not block:
                if not self._qsize():
    
                  raise Empty
    

    E _queue.Empty

    /usr/local/lib/python3.8/queue.py:167: Empty

    During handling of the above exception, another exception occurred:

    args = (database: "projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_database-1629761046672"
    session {
    labels {
    key: "testcase"
    value: "database_api"
    }
    }
    ,)
    kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629761023546/databa...46/databases/test_database-1629761046672'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
    
          return callable_(*args, **kwargs)
    

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:66:


    self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f24d91fc880>
    request = database: "projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_database-1629761046672"
    session {
    labels {
    key: "testcase"
    value: "database_api"
    }
    }

    timeout = None
    metadata = [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_data...546/databases/test_database-1629761046672'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]
    credentials = None, wait_for_ready = None, compression = None

    def __call__(self,
                 request,
                 timeout=None,
                 metadata=None,
                 credentials=None,
                 wait_for_ready=None,
                 compression=None):
        state, call, = self._blocking(request, timeout, metadata, credentials,
                                      wait_for_ready, compression)
    
      return _end_unary_response_blocking(state, call, False, None)
    

    .nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:


    state = <grpc._channel._RPCState object at 0x7f24d5a1ea00>
    call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f24d55d9bc0>
    with_call = False, deadline = None

    def _end_unary_response_blocking(state, call, with_call, deadline):
        if state.code is grpc.StatusCode.OK:
            if with_call:
                rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
                return state.response, rendezvous
            else:
                return state.response
        else:
    
          raise _InactiveRpcError(state)
    

    E grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:
    E status = StatusCode.NOT_FOUND
    E details = "Instance not found: projects/precise-truck-742/instances/google-cloud-1629761023546"
    E debug_error_string = "{"created":"@1629766155.774203044","description":"Error received from peer ipv4:74.125.142.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Instance not found: projects/precise-truck-742/instances/google-cloud-1629761023546","grpc_status":5}"
    E >

    .nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError

    The above exception was the direct cause of the following exception:

    shared_database = <google.cloud.spanner_v1.database.Database object at 0x7f24d9075e20>

    def test_table_exists(shared_database):
        table = shared_database.table("all_types")
    
      assert table.exists()
    

    tests/system/test_table_api.py:23:


    google/cloud/spanner_v1/table.py:67: in exists
    with self._database.snapshot() as snapshot:
    google/cloud/spanner_v1/database.py:855: in enter
    session = self._session = self._database._pool.get()
    google/cloud/spanner_v1/pool.py:275: in get
    session.create()
    google/cloud/spanner_v1/session.py:126: in create
    session_pb = api.create_session(request=request, metadata=metadata,)
    google/cloud/spanner_v1/services/spanner/client.py:459: in create_session
    response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)
    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:142: in call
    return wrapped_func(*args, **kwargs)
    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283: in retry_wrapped_func
    return retry_target(
    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190: in retry_target
    return target()


    args = (database: "projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_database-1629761046672"
    session {
    labels {
    key: "testcase"
    value: "database_api"
    }
    }
    ,)
    kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629761023546/databa...46/databases/test_database-1629761046672'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]}

    @functools.wraps(callable_)
    def error_remapped_callable(*args, **kwargs):
        try:
            return callable_(*args, **kwargs)
        except grpc.RpcError as exc:
    
          raise exceptions.from_grpc_error(exc) from exc
    

    E google.api_core.exceptions.NotFound: 404 Instance not found: projects/precise-truck-742/instances/google-cloud-1629761023546

    .nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:68: NotFound

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

Labels

🚨This issue needs some love.api: spannerIssues related to the googleapis/python-spanner API.flakybot: flakyTells the Flaky Bot not to close or comment on this issue.flakybot: issueAn issue filed by the Flaky Bot. Should not be added manually.priority: p1Important issue which blocks shipping the next release. Will be fixed prior to next release.type: bugError or flaw in code with unintended results or allowing sub-optimal usage patterns.

Type

No type

Projects

No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions