tests.system.test_backup_api: test_instance_list_backups failed

This test failed!

To configure my behavior, see the Flaky Bot documentation.

If I'm commenting on this issue too often, add the flakybot: quiet label and
I will stop commenting.

commit: bdd5f8b
buildURL: Build Status, Sponge
status: failed

Test output

target = functools.partial(>)
predicate = .if_exception_type_predicate at 0x7f6d8182d3a0>
sleep_generator = 
deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:


          return target()


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:

self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>

retry = <google.api_core.retry.Retry object at 0x7f6d81831430>
def _done_or_raise(self, retry=DEFAULT_RETRY):
    """Check if the future is done and raise if it's not."""
    kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

    if not self.done(**kwargs):


      raise _OperationNotComplete()


E           google.api_core.future.polling._OperationNotComplete
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete
The above exception was the direct cause of the following exception:
self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>

timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f6d81831430>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}


      retry_(self._done_or_raise)(**kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:

args = (), kwargs = {}

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>)

sleep_generator = <generator object exponential_sleep_generator at 0x7f6d812e1510>
@general_helpers.wraps(func)
def retry_wrapped_func(*args, **kwargs):
    """A wrapper that calls target function with retry."""
    target = functools.partial(func, *args, **kwargs)
    sleep_generator = exponential_sleep_generator(
        self._initial, self._maximum, multiplier=self._multiplier
    )


  return retry_target(


        target,
        self._predicate,
        sleep_generator,
        self._deadline,
        on_error=on_error,
    )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>)

predicate = <function if_exception_type..if_exception_type_predicate at 0x7f6d8182d3a0>

sleep_generator = <generator object exponential_sleep_generator at 0x7f6d812e1510>

deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
            return target()

        # pylint: disable=broad-except
        # This function explicitly must deal with broad exceptions.
        except Exception as exc:
            if not predicate(exc):
                raise
            last_exc = exc
            if on_error is not None:
                on_error(exc)

        now = datetime_helpers.utcnow()

        if deadline_datetime is not None:
            if deadline_datetime <= now:


              six.raise_from(


                    exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
                    ),
                    last_exc,
                )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:

value = None, from_value = _OperationNotComplete()

???

E   google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7f6d811acaf0>>), last exception:

:3: RetryError
During handling of the above exception, another exception occurred:
spanner_client = <google.cloud.spanner_v1.client.Client object at 0x7f6d811926d0>

operation_timeout = 60, shared_instance_id = 'google-cloud-1628986419564'

instance_config = name: "projects/precise-truck-742/instanceConfigs/regional-us-central1"

display_name: "us-central1"

replicas {

locat...

replicas {

location: "us-central1"

type_: READ_WRITE

}

replicas {

location: "us-central1"

type_: READ_WRITE

}
existing_instances = [name: "projects/precise-truck-742/instances/gcp-streaming-systests"

config: "projects/precise-truck-742/instanceConfi...ated"

value: "1628986209"

}

labels {

key: "python-spanner-systests"

value: "true"

}

processing_units: 1000

, ...]
@pytest.fixture(scope="session")
def shared_instance(
    spanner_client,
    operation_timeout,
    shared_instance_id,
    instance_config,
    existing_instances,  # evalutate before creating one
):
    _helpers.cleanup_old_instances(spanner_client)

    if _helpers.CREATE_INSTANCE:
        create_time = str(int(time.time()))
        labels = {"python-spanner-systests": "true", "created": create_time}

        instance = spanner_client.instance(
            shared_instance_id, instance_config.name, labels=labels
        )
        created_op = _helpers.retry_429_503(instance.create)()


      created_op.result(operation_timeout)  # block until completion


tests/system/conftest.py:119:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

self = <google.api_core.operation.Operation object at 0x7f6d811acaf0>

timeout = 60, retry = <google.api_core.retry.Retry object at 0x7f6d81831430>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
        retry_(self._done_or_raise)(**kwargs)
    except exceptions.RetryError:


      raise concurrent.futures.TimeoutError(


            "Operation did not complete within the designated " "timeout."
        )

E           concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

commit: 372b40d
buildURL: Build Status, Sponge
status: failed

Test output

args = (parent: "projects/precise-truck-742/instances/google-cloud-1629010600345"
backup_id: "backup_id1_1629015310046"
backu...
    seconds: 1630829710
    nanos: 46571000
  }
  version_time {
    seconds: 1629010603
    nanos: 629238000
  }
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629010600345/backup...42/instances/google-cloud-1629010600345'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/1.31.1 gccl/3.7.0')]}
@six.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:67:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7ff0aae381c0>

request = parent: "projects/precise-truck-742/instances/google-cloud-1629010600345"

backup_id: "backup_id1_1629015310046"

backup... {

seconds: 1630829710

nanos: 46571000

}

version_time {

seconds: 1629010603

nanos: 629238000

}

}
timeout = None

metadata = [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629010600345/backups/backup_id1_...742/instances/google-cloud-1629010600345'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/1.31.1 gccl/3.7.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7ff0aad10940>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7ff0aad1ad40>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.FAILED_PRECONDITION

E           	details = "Read-only transaction timestamp 2021-08-15T06:56:43.629238Z has exceeded the maximum timestamp staleness"

E           	debug_error_string = "{"created":"@1629015310.245183942","description":"Error received from peer ipv4:74.125.142.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Read-only transaction timestamp 2021-08-15T06:56:43.629238Z has exceeded the maximum timestamp staleness","grpc_status":9}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
shared_instance = <google.cloud.spanner_v1.instance.Instance object at 0x7ff0aae339d0>

shared_database = <google.cloud.spanner_v1.database.Database object at 0x7ff0aae3c730>

second_database = <google.cloud.spanner_v1.database.Database object at 0x7ff0aaf9c2e0>

database_version_time = datetime.datetime(2021, 8, 15, 6, 56, 43, 629238, tzinfo=datetime.timezone.utc)

backups_to_delete = []
def test_instance_list_backups(
    shared_instance,
    shared_database,
    second_database,
    database_version_time,
    backups_to_delete,
):
    # Remove un-scrubbed backups FBO count below.
    _helpers.scrub_instance_backups(shared_instance)

    backup_id_1 = _helpers.unique_id("backup_id1", separator="_")
    backup_id_2 = _helpers.unique_id("backup_id2", separator="_")

    expire_time_1 = datetime.datetime.utcnow() + datetime.timedelta(days=21)
    expire_time_1 = expire_time_1.replace(tzinfo=datetime.timezone.utc)
    expire_time_1_stamp = expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    backup1 = shared_instance.backup(
        backup_id_1,
        database=shared_database,
        expire_time=expire_time_1,
        version_time=database_version_time,
    )

    expire_time_2 = datetime.datetime.utcnow() + datetime.timedelta(days=1)
    expire_time_2 = expire_time_2.replace(tzinfo=datetime.timezone.utc)
    backup2 = shared_instance.backup(
        backup_id_2, database=second_database, expire_time=expire_time_2
    )

    # Create two backups.


  op1 = backup1.create()


tests/system/test_backup_api.py:407:

google/cloud/spanner_v1/backup.py:261: in create

future = api.create_backup(request=request, metadata=metadata,)

google/cloud/spanner_admin_database_v1/services/database_admin/client.py:1430: in create_backup

response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:145: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:69: in error_remapped_callable

six.raise_from(exceptions.from_grpc_error(exc), exc)

value = None

from_value = <_InactiveRpcError of RPC that terminated with:

status = StatusCode.FAILED_PRECONDITION

details = "Read-only transac...nly transaction timestamp 2021-08-15T06:56:43.629238Z has exceeded the maximum timestamp staleness","grpc_status":9}"



???

E   google.api_core.exceptions.FailedPrecondition: 400 Read-only transaction timestamp 2021-08-15T06:56:43.629238Z has exceeded the maximum timestamp staleness

:3: FailedPrecondition

Looks like this issue is flaky. 😟

I'm going to leave this open and stop commenting.

A human should fix and close this.

When run at the same commit (372b40d), this test passed in one build (Build Status, Sponge) and failed in another build (Build Status, Sponge).

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

commit: 492ad09
buildURL: Build Status, Sponge
status: failed

Test output

target = functools.partial(>)
predicate = .if_exception_type_predicate at 0x7fdd86b883a0>
sleep_generator = 
deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:


          return target()


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:189:

self = <google.api_core.operation.Operation object at 0x7fdd864f9520>

retry = <google.api_core.retry.Retry object at 0x7fdd86b87b80>
def _done_or_raise(self, retry=DEFAULT_RETRY):
    """Check if the future is done and raise if it's not."""
    kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}

    if not self.done(**kwargs):


      raise _OperationNotComplete()


E           google.api_core.future.polling._OperationNotComplete
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:87: _OperationNotComplete
The above exception was the direct cause of the following exception:
self = <google.api_core.operation.Operation object at 0x7fdd864f9520>

timeout = 60, retry = <google.api_core.retry.Retry object at 0x7fdd86b87b80>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}


      retry_(self._done_or_raise)(**kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:108:

args = (), kwargs = {}

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fdd864f9520>>)

sleep_generator = <generator object exponential_sleep_generator at 0x7fdd866453c0>
@general_helpers.wraps(func)
def retry_wrapped_func(*args, **kwargs):
    """A wrapper that calls target function with retry."""
    target = functools.partial(func, *args, **kwargs)
    sleep_generator = exponential_sleep_generator(
        self._initial, self._maximum, multiplier=self._multiplier
    )


  return retry_target(


        target,
        self._predicate,
        sleep_generator,
        self._deadline,
        on_error=on_error,
    )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:286:

target = functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fdd864f9520>>)

predicate = <function if_exception_type..if_exception_type_predicate at 0x7fdd86b883a0>

sleep_generator = <generator object exponential_sleep_generator at 0x7fdd866453c0>

deadline = 60, on_error = None
def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
    """Call a function and retry if it fails.

    This is the lowest-level retry helper. Generally, you'll use the
    higher-level retry helper :class:`Retry`.

    Args:
        target(Callable): The function to call and retry. This must be a
            nullary function - apply arguments with `functools.partial`.
        predicate (Callable[Exception]): A callable used to determine if an
            exception raised by the target should be considered retryable.
            It should return True to retry or False otherwise.
        sleep_generator (Iterable[float]): An infinite iterator that determines
            how long to sleep between retries.
        deadline (float): How long to keep retrying the target. The last sleep
            period is shortened as necessary, so that the last retry runs at
            ``deadline`` (and not considerably beyond it).
        on_error (Callable[Exception]): A function to call while processing a
            retryable exception.  Any error raised by this function will *not*
            be caught.

    Returns:
        Any: the return value of the target function.

    Raises:
        google.api_core.RetryError: If the deadline is exceeded while retrying.
        ValueError: If the sleep generator stops yielding values.
        Exception: If the target raises a method that isn't retryable.
    """
    if deadline is not None:
        deadline_datetime = datetime_helpers.utcnow() + datetime.timedelta(
            seconds=deadline
        )
    else:
        deadline_datetime = None

    last_exc = None

    for sleep in sleep_generator:
        try:
            return target()

        # pylint: disable=broad-except
        # This function explicitly must deal with broad exceptions.
        except Exception as exc:
            if not predicate(exc):
                raise
            last_exc = exc
            if on_error is not None:
                on_error(exc)

        now = datetime_helpers.utcnow()

        if deadline_datetime is not None:
            if deadline_datetime <= now:


              six.raise_from(


                    exceptions.RetryError(
                        "Deadline of {:.1f}s exceeded while calling {}".format(
                            deadline, target
                        ),
                        last_exc,
                    ),
                    last_exc,
                )

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:204:

value = None, from_value = _OperationNotComplete()

???

E   google.api_core.exceptions.RetryError: Deadline of 60.0s exceeded while calling functools.partial(<bound method PollingFuture._done_or_raise of <google.api_core.operation.Operation object at 0x7fdd864f9520>>), last exception:

:3: RetryError
During handling of the above exception, another exception occurred:
spanner_client = <google.cloud.spanner_v1.client.Client object at 0x7fdd864e6be0>

operation_timeout = 60, shared_instance_id = 'google-cloud-1629113513103'

instance_config = name: "projects/precise-truck-742/instanceConfigs/regional-us-central1"

display_name: "us-central1"

replicas {

locat...

replicas {

location: "us-central1"

type_: READ_WRITE

}

replicas {

location: "us-central1"

type_: READ_WRITE

}
existing_instances = [name: "projects/precise-truck-742/instances/gcp-streaming-systests"

config: "projects/precise-truck-742/instanceConfi...: "created"

value: "1627588424"

}

labels {

key: "sample_name"

value: "autocommit"

}

processing_units: 1000

, ...]
@pytest.fixture(scope="session")
def shared_instance(
    spanner_client,
    operation_timeout,
    shared_instance_id,
    instance_config,
    existing_instances,  # evalutate before creating one
):
    _helpers.cleanup_old_instances(spanner_client)

    if _helpers.CREATE_INSTANCE:
        create_time = str(int(time.time()))
        labels = {"python-spanner-systests": "true", "created": create_time}

        instance = spanner_client.instance(
            shared_instance_id, instance_config.name, labels=labels
        )
        created_op = _helpers.retry_429_503(instance.create)()


      created_op.result(operation_timeout)  # block until completion


tests/system/conftest.py:119:

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:130: in result

self._blocking_poll(timeout=timeout, **kwargs)

self = <google.api_core.operation.Operation object at 0x7fdd864f9520>

timeout = 60, retry = <google.api_core.retry.Retry object at 0x7fdd86b87b80>
def _blocking_poll(self, timeout=None, retry=DEFAULT_RETRY):
    """Poll and wait for the Future to be resolved.

    Args:
        timeout (int):
            How long (in seconds) to wait for the operation to complete.
            If None, wait indefinitely.
    """
    if self._result_set:
        return

    retry_ = self._retry.with_deadline(timeout)

    try:
        kwargs = {} if retry is DEFAULT_RETRY else {"retry": retry}
        retry_(self._done_or_raise)(**kwargs)
    except exceptions.RetryError:


      raise concurrent.futures.TimeoutError(


            "Operation did not complete within the designated " "timeout."
        )

E           concurrent.futures._base.TimeoutError: Operation did not complete within the designated timeout.
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/future/polling.py:110: TimeoutError

Closed by #525

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

commit: b294dbf
buildURL: Build Status, Sponge
status: failed

Test output

args = (parent: "projects/precise-truck-742/instances/google-cloud-1629350036171"
backup_id: "backup_id1_1629354413267"
backu...    seconds: 1631168813
    nanos: 267944000
  }
  version_time {
    seconds: 1629350041
    nanos: 401595000
  }
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629350036171/backup...742/instances/google-cloud-1629350036171'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]}
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:66:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f24539015b0>

request = parent: "projects/precise-truck-742/instances/google-cloud-1629350036171"

backup_id: "backup_id1_1629354413267"

backup...{

seconds: 1631168813

nanos: 267944000

}

version_time {

seconds: 1629350041

nanos: 401595000

}

}
timeout = None

metadata = [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629350036171/backups/backup_id1_...-742/instances/google-cloud-1629350036171'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f2453a6cc40>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f2453a50f80>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.FAILED_PRECONDITION

E           	details = "Precondition check failed."

E           	debug_error_string = "{"created":"@1629354413.456674264","description":"Error received from peer ipv4:74.125.142.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Precondition check failed.","grpc_status":9}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
shared_instance = <google.cloud.spanner_v1.instance.Instance object at 0x7f245383c610>

shared_database = <google.cloud.spanner_v1.database.Database object at 0x7f245390d400>

second_database = <google.cloud.spanner_v1.database.Database object at 0x7f245384a790>

database_version_time = datetime.datetime(2021, 8, 19, 5, 14, 1, 401595, tzinfo=datetime.timezone.utc)

backups_to_delete = []
def test_instance_list_backups(
    shared_instance,
    shared_database,
    second_database,
    database_version_time,
    backups_to_delete,
):
    # Remove un-scrubbed backups FBO count below.
    _helpers.scrub_instance_backups(shared_instance)

    backup_id_1 = _helpers.unique_id("backup_id1", separator="_")
    backup_id_2 = _helpers.unique_id("backup_id2", separator="_")

    expire_time_1 = datetime.datetime.utcnow() + datetime.timedelta(days=21)
    expire_time_1 = expire_time_1.replace(tzinfo=datetime.timezone.utc)
    expire_time_1_stamp = expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    backup1 = shared_instance.backup(
        backup_id_1,
        database=shared_database,
        expire_time=expire_time_1,
        version_time=database_version_time,
    )

    expire_time_2 = datetime.datetime.utcnow() + datetime.timedelta(days=1)
    expire_time_2 = expire_time_2.replace(tzinfo=datetime.timezone.utc)
    backup2 = shared_instance.backup(
        backup_id_2, database=second_database, expire_time=expire_time_2
    )

    # Create two backups.


  op1 = backup1.create()


tests/system/test_backup_api.py:406:

google/cloud/spanner_v1/backup.py:261: in create

future = api.create_backup(request=request, metadata=metadata,)

google/cloud/spanner_admin_database_v1/services/database_admin/client.py:1430: in create_backup

response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:142: in call

return wrapped_func(*args, **kwargs)

args = (parent: "projects/precise-truck-742/instances/google-cloud-1629350036171"

backup_id: "backup_id1_1629354413267"

backu...    seconds: 1631168813

nanos: 267944000

}

version_time {

seconds: 1629350041

nanos: 401595000

}

}

,)

kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629350036171/backup...742/instances/google-cloud-1629350036171'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]}
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
        return callable_(*args, **kwargs)
    except grpc.RpcError as exc:


      raise exceptions.from_grpc_error(exc) from exc


E           google.api_core.exceptions.FailedPrecondition: 400 Precondition check failed.
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:68: FailedPrecondition

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

commit: 098a786
buildURL: Build Status, Sponge
status: failed

Test output

args = (parent: "projects/precise-truck-742/instances/google-cloud-1629761023546"
backup_id: "backup_id1_1629766022104"
backu...
    seconds: 1631580422
    nanos: 104336000
  }
  version_time {
    seconds: 1629766022
    nanos: 50785000
  }
}
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629761023546/backup...742/instances/google-cloud-1629761023546'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]}
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:66:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7f24d8fb14f0>

request = parent: "projects/precise-truck-742/instances/google-cloud-1629761023546"

backup_id: "backup_id1_1629766022104"

backup... {

seconds: 1631580422

nanos: 104336000

}

version_time {

seconds: 1629766022

nanos: 50785000

}

}
timeout = None

metadata = [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629761023546/backups/backup_id1_...-742/instances/google-cloud-1629761023546'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7f24d8fc3be0>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7f24d9123480>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.NOT_FOUND

E           	details = "Database not found: projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_database-1629761046672"

E           	debug_error_string = "{"created":"@1629766022.155319278","description":"Error received from peer ipv4:74.125.195.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Database not found: projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_database-1629761046672","grpc_status":5}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
shared_instance = <google.cloud.spanner_v1.instance.Instance object at 0x7f24d909f880>

shared_database = <google.cloud.spanner_v1.database.Database object at 0x7f24d9075e20>

second_database = <google.cloud.spanner_v1.database.Database object at 0x7f24d8fc3760>

database_version_time = datetime.datetime(2021, 8, 24, 0, 47, 2, 50785, tzinfo=datetime.timezone.utc)

backups_to_delete = []
def test_instance_list_backups(
    shared_instance,
    shared_database,
    second_database,
    database_version_time,
    backups_to_delete,
):
    # Remove un-scrubbed backups FBO count below.
    _helpers.scrub_instance_backups(shared_instance)

    backup_id_1 = _helpers.unique_id("backup_id1", separator="_")
    backup_id_2 = _helpers.unique_id("backup_id2", separator="_")

    expire_time_1 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
        days=21
    )
    expire_time_1_stamp = expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ")

    backup1 = shared_instance.backup(
        backup_id_1,
        database=shared_database,
        expire_time=expire_time_1,
        version_time=database_version_time,
    )

    expire_time_2 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
        days=1
    )
    backup2 = shared_instance.backup(
        backup_id_2, database=second_database, expire_time=expire_time_2
    )

    # Create two backups.


  op1 = backup1.create()


tests/system/test_backup_api.py:413:

google/cloud/spanner_v1/backup.py:261: in create

future = api.create_backup(request=request, metadata=metadata,)

google/cloud/spanner_admin_database_v1/services/database_admin/client.py:1430: in create_backup

response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:142: in call

return wrapped_func(*args, **kwargs)

args = (parent: "projects/precise-truck-742/instances/google-cloud-1629761023546"

backup_id: "backup_id1_1629766022104"

backu...

seconds: 1631580422

nanos: 104336000

}

version_time {

seconds: 1629766022

nanos: 50785000

}

}

,)

kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1629761023546/backup...742/instances/google-cloud-1629761023546'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.39.0 gax/2.0.0 gccl/3.8.0')]}
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
        return callable_(*args, **kwargs)
    except grpc.RpcError as exc:


      raise exceptions.from_grpc_error(exc) from exc


E           google.api_core.exceptions.NotFound: 404 Database not found: projects/precise-truck-742/instances/google-cloud-1629761023546/databases/test_database-1629761046672
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:68: NotFound

Oops! Looks like this issue is still flaky. It failed again. 😬

I reopened the issue, but a human will need to close it again.

commit: db63aee
buildURL: Build Status, Sponge
status: failed

Test output

args = (name: "projects/precise-truck-742/instances/google-cloud-1633384344484/backups/backup_id1_1633389214004"
,)
kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1633384344484/backup...344484/backups/backup_id1_1633389214004'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.41.0 gax/2.0.1 gccl/3.11.0')]}
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:


      return callable_(*args, **kwargs)


.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:66:

self = <grpc._channel._UnaryUnaryMultiCallable object at 0x7fea37e988e0>

request = name: "projects/precise-truck-742/instances/google-cloud-1633384344484/backups/backup_id1_1633389214004"
timeout = None

metadata = [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1633384344484/backups/backup_id1_...4344484/backups/backup_id1_1633389214004'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.41.0 gax/2.0.1 gccl/3.11.0')]

credentials = None, wait_for_ready = None, compression = None
def __call__(self,
             request,
             timeout=None,
             metadata=None,
             credentials=None,
             wait_for_ready=None,
             compression=None):
    state, call, = self._blocking(request, timeout, metadata, credentials,
                                  wait_for_ready, compression)


  return _end_unary_response_blocking(state, call, False, None)


.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:946:

state = <grpc._channel._RPCState object at 0x7fea3808ca90>

call = <grpc._cython.cygrpc.SegregatedCall object at 0x7fea3819d480>

with_call = False, deadline = None
def _end_unary_response_blocking(state, call, with_call, deadline):
    if state.code is grpc.StatusCode.OK:
        if with_call:
            rendezvous = _MultiThreadedRendezvous(state, call, None, deadline)
            return state.response, rendezvous
        else:
            return state.response
    else:


      raise _InactiveRpcError(state)


E           grpc._channel._InactiveRpcError: <_InactiveRpcError of RPC that terminated with:

E           	status = StatusCode.NOT_FOUND

E           	details = "Instance not found: projects/precise-truck-742/instances/google-cloud-1633384344484"

E           	debug_error_string = "{"created":"@1633389777.236909176","description":"Error received from peer ipv4:74.125.195.95:443","file":"src/core/lib/surface/call.cc","file_line":1069,"grpc_message":"Instance not found: projects/precise-truck-742/instances/google-cloud-1633384344484","grpc_status":5}"

E           >
.nox/system-3-8/lib/python3.8/site-packages/grpc/_channel.py:849: _InactiveRpcError
The above exception was the direct cause of the following exception:
@pytest.fixture(scope="function")
def backups_to_delete():
    to_delete = []

    yield to_delete

    for backup in to_delete:


      _helpers.retry_429_503(backup.delete)()


tests/system/test_backup_api.py:114:

.nox/system-3-8/lib/python3.8/site-packages/test_utils/retry.py:100: in wrapped_function

return to_wrap(*args, **kwargs)

google/cloud/spanner_v1/backup.py:325: in delete

api.delete_backup(name=self.name, metadata=metadata)

google/cloud/spanner_admin_database_v1/services/database_admin/client.py:1666: in delete_backup

rpc(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/gapic_v1/method.py:142: in call

return wrapped_func(*args, **kwargs)

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:283: in retry_wrapped_func

return retry_target(

.nox/system-3-8/lib/python3.8/site-packages/google/api_core/retry.py:190: in retry_target

return target()

args = (name: "projects/precise-truck-742/instances/google-cloud-1633384344484/backups/backup_id1_1633389214004"

,)

kwargs = {'metadata': [('google-cloud-resource-prefix', 'projects/precise-truck-742/instances/google-cloud-1633384344484/backup...344484/backups/backup_id1_1633389214004'), ('x-goog-api-client', 'gl-python/3.8.6 grpc/1.41.0 gax/2.0.1 gccl/3.11.0')]}
@functools.wraps(callable_)
def error_remapped_callable(*args, **kwargs):
    try:
        return callable_(*args, **kwargs)
    except grpc.RpcError as exc:


      raise exceptions.from_grpc_error(exc) from exc


E           google.api_core.exceptions.NotFound: 404 Instance not found: projects/precise-truck-742/instances/google-cloud-1633384344484
.nox/system-3-8/lib/python3.8/site-packages/google/api_core/grpc_helpers.py:68: NotFound

Navigation Menu

Repository navigation

tests.system.test_backup_api: test_instance_list_backups failed #496

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

tests.system.test_backup_api: test_instance_list_backups failed #496

Description

Activity

flaky-bot commented on Aug 15, 2021

flaky-bot commented on Aug 15, 2021

flaky-bot commented on Aug 16, 2021

larkee commented on Aug 18, 2021

flaky-bot commented on Aug 19, 2021

flaky-bot commented on Aug 24, 2021

flaky-bot commented on Oct 4, 2021

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions