From ef67fbd29234b20de39157e33efdae5e7789600b Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 11 May 2022 14:20:34 -0400 Subject: [PATCH 01/30] remove status from application latency --- .../google/cloud/bigtable/stats/StatsRecorderWrapperTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java index abf00e71b340..11e9184e10e9 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java @@ -92,7 +92,6 @@ public void testStreamingOperation() throws InterruptedException { recorderWrapper.putGfeMissingHeaders(connectivityErrorCount); recorderWrapper.putFirstResponseLatencies(firstResponseLatency); recorderWrapper.putBatchRequestThrottled(throttlingLatency); - recorderWrapper.record("OK", TABLE_ID, ZONE, CLUSTER); Thread.sleep(100); From 8c943fb65c59ee5f2d6c4bdcc298ab61740c0da7 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 10 May 2022 11:49:36 -0400 Subject: [PATCH 02/30] feat: update tracers to use built in metrics --- google-cloud-bigtable/pom.xml | 1 - pom.xml | 11 ++++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index e5a849629573..e0f7b6f10d92 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -60,7 +60,6 @@ - com.google.cloud google-cloud-bigtable-stats diff --git a/pom.xml b/pom.xml index 11db087d5ef8..908ac282f333 100644 --- a/pom.xml +++ b/pom.xml @@ -212,6 +212,15 @@ + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + @@ -343,11 +352,11 @@ google-cloud-bigtable + google-cloud-bigtable-stats grpc-google-cloud-bigtable-admin-v2 grpc-google-cloud-bigtable-v2 proto-google-cloud-bigtable-admin-v2 proto-google-cloud-bigtable-v2 - google-cloud-bigtable-stats google-cloud-bigtable-emulator-core google-cloud-bigtable-emulator google-cloud-bigtable-bom From 8253a9a5b102c9f4f54f2cdb47497d03f8b10ea6 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 8 Sep 2021 16:06:07 -0400 Subject: [PATCH 03/30] feat: add response protos --- google-cloud-bigtable-stats/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index f16f2c211bdc..f14a7292b6e9 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -48,7 +48,6 @@ com.google.api api-common - io.opencensus opencensus-api From 92b1f48ce04bdcfb5c7a6afe4a50272fd7b129bf Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 8 Sep 2021 16:06:07 -0400 Subject: [PATCH 04/30] feat: add response protos --- google-cloud-bigtable-stats/pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index f14a7292b6e9..af8aeb7496d6 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -48,10 +48,29 @@ com.google.api api-common + + com.google.cloud + google-cloud-monitoring + 3.2.9 + io.opencensus opencensus-api + + io.opencensus + opencensus-impl + + + io.opencensus + opencensus-exporter-stats-stackdriver + + + com.google.cloud + google-cloud-monitoring + + + com.google.guava guava From 50a1eab5e502144b548f0b46df4befdba4aa2fdb Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 12 May 2022 14:40:40 -0400 Subject: [PATCH 05/30] feat: add stackdriver exporter --- google-cloud-bigtable-stats/pom.xml | 10 + .../BigtableCreateTimeSeriesExporter.java | 124 +++++++ .../BigtableStackdriverExportUtils.java | 334 ++++++++++++++++++ .../BigtableStackdriverStatsExporter.java | 113 ++++++ .../BigtableCreateTimeSeriesExporterTest.java | 146 ++++++++ .../data/v2/BigtableDataSettings.java | 41 +++ 6 files changed, 768 insertions(+) create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java create mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index af8aeb7496d6..23e20d4b2488 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -91,6 +91,16 @@ junit test + + io.grpc + grpc-api + test + + + org.mockito + mockito-core + test + diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java new file mode 100644 index 000000000000..e66c86613910 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java @@ -0,0 +1,124 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats.exporter; + +import com.google.api.MonitoredResource; +import com.google.api.gax.rpc.ApiException; +import com.google.cloud.bigtable.stats.BuiltinMeasureConstants; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.monitoring.v3.ProjectName; +import com.google.monitoring.v3.TimeSeries; +import io.opencensus.exporter.metrics.util.MetricExporter; +import io.opencensus.metrics.LabelKey; +import io.opencensus.metrics.LabelValue; +import io.opencensus.metrics.export.Metric; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +final class BigtableCreateTimeSeriesExporter extends MetricExporter { + private static final Logger logger = + Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); + private final ProjectName projectName; + private final MetricServiceClient metricServiceClient; + private final MonitoredResource monitoredResource; + private final String domain; + + BigtableCreateTimeSeriesExporter( + String projectId, + MetricServiceClient metricServiceClient, + MonitoredResource monitoredResource) { + this.projectName = ProjectName.newBuilder().setProject(projectId).build(); + this.metricServiceClient = metricServiceClient; + this.monitoredResource = monitoredResource; + this.domain = "bigtable.googleapis.com/client/"; + } + + public void export(Collection metrics) { + List timeSeriesList = new ArrayList(metrics.size()); + + for (Metric metric : metrics) { + // only export bigtable metrics + if (!metric.getMetricDescriptor().getName().contains("bigtable")) { + continue; + } + + for (io.opencensus.metrics.export.TimeSeries timeSeries : metric.getTimeSeriesList()) { + MonitoredResource.Builder monitoredResourceBuilder = this.monitoredResource.toBuilder(); + + List keys = metric.getMetricDescriptor().getLabelKeys(); + List labelValues = timeSeries.getLabelValues(); + + List updatedKeys = new ArrayList<>(); + List updatedValues = new ArrayList<>(); + + for (int i = 0; i < labelValues.size(); i++) { + if (keys.get(i).getKey().equals(BuiltinMeasureConstants.PROJECT_ID.getName())) { + monitoredResourceBuilder.putLabels( + BuiltinMeasureConstants.PROJECT_ID.getName(), labelValues.get(i).getValue()); + } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.INSTANCE_ID.getName())) { + monitoredResourceBuilder.putLabels( + BuiltinMeasureConstants.INSTANCE_ID.getName(), labelValues.get(i).getValue()); + } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.CLUSTER.getName())) { + monitoredResourceBuilder.putLabels( + BuiltinMeasureConstants.CLUSTER.getName(), labelValues.get(i).getValue()); + } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.ZONE.getName())) { + monitoredResourceBuilder.putLabels( + BuiltinMeasureConstants.ZONE.getName(), labelValues.get(i).getValue()); + } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.TABLE.getName())) { + monitoredResourceBuilder.putLabels( + BuiltinMeasureConstants.TABLE.getName(), labelValues.get(i).getValue()); + } else { + updatedKeys.add(keys.get(i)); + updatedValues.add(labelValues.get(i)); + } + } + + updatedKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_ID.getName(), "client id")); + updatedValues.add( + LabelValue.create(BigtableStackdriverExportUtils.generateDefaultTaskValue())); + + timeSeriesList.add( + BigtableStackdriverExportUtils.convertTimeSeries( + metric.getMetricDescriptor().getName(), + metric.getMetricDescriptor().getType(), + updatedKeys, + updatedValues, + timeSeries, + monitoredResourceBuilder.build(), + this.domain, + this.projectName.getProject())); + } + } + + try { + CreateTimeSeriesRequest request = + CreateTimeSeriesRequest.newBuilder() + .setName(this.projectName.toString()) + .addAllTimeSeries(timeSeriesList) + .build(); + + this.metricServiceClient.createServiceTimeSeries(request); + } catch (ApiException e) { + logger.log(Level.WARNING, "ApiException thrown when exporting TimeSeries.", e); + } catch (Throwable e) { + logger.log(Level.WARNING, "Exception thrown when exporting TimeSeries.", e); + } + } +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java new file mode 100644 index 000000000000..f0fc1118ad85 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java @@ -0,0 +1,334 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats.exporter; + +import com.google.api.Distribution.BucketOptions; +import com.google.api.Distribution.BucketOptions.Explicit; +import com.google.api.MetricDescriptor.MetricKind; +import com.google.api.MonitoredResource; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import com.google.monitoring.v3.TimeInterval; +import com.google.monitoring.v3.TimeSeries; +import com.google.monitoring.v3.TypedValue; +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import io.opencensus.common.Function; +import io.opencensus.common.Functions; +import io.opencensus.contrib.exemplar.util.AttachmentValueSpanContext; +import io.opencensus.contrib.resource.util.ResourceUtils; +import io.opencensus.metrics.LabelKey; +import io.opencensus.metrics.LabelValue; +import io.opencensus.metrics.data.AttachmentValue; +import io.opencensus.metrics.export.Distribution.Bucket; +import io.opencensus.metrics.export.Distribution.BucketOptions.ExplicitOptions; +import io.opencensus.metrics.export.MetricDescriptor.Type; +import io.opencensus.metrics.export.Summary; +import io.opencensus.metrics.export.Value; +import io.opencensus.resource.Resource; + +import java.lang.management.ManagementFactory; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.security.SecureRandom; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +class BigtableStackdriverExportUtils { + private static final Logger logger; + + private static final io.opencensus.common.Function typedValueDoubleFunction; + private static final io.opencensus.common.Function typedValueLongFunction; + private static final io.opencensus.common.Function< + io.opencensus.metrics.export.Distribution, TypedValue> + typedValueDistributionFunction; + private static final io.opencensus.common.Function typedValueSummaryFunction; + private static final io.opencensus.common.Function + bucketOptionsExplicitFunction; + + static String generateDefaultTaskValue() { + String jvmName = ManagementFactory.getRuntimeMXBean().getName(); + if (jvmName.indexOf(64) < 1) { + String hostname = "localhost"; + + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + logger.log(Level.INFO, "Unable to get the hostname.", e); + } + + return "java-" + (new SecureRandom()).nextInt() + "@" + hostname; + } else { + return "java-" + jvmName; + } + } + + private static String generateType(String metricName, String domain) { + return domain + metricName; + } + + @VisibleForTesting + static MetricKind createMetricKind(Type type) { + if (type != Type.GAUGE_INT64 && type != Type.GAUGE_DOUBLE) { + return type != Type.CUMULATIVE_INT64 + && type != Type.CUMULATIVE_DOUBLE + && type != Type.CUMULATIVE_DISTRIBUTION + ? MetricKind.UNRECOGNIZED + : MetricKind.CUMULATIVE; + } else { + return MetricKind.GAUGE; + } + } + + @VisibleForTesting + static com.google.api.MetricDescriptor.ValueType createValueType(Type type) { + if (type != Type.CUMULATIVE_DOUBLE && type != Type.GAUGE_DOUBLE) { + if (type != Type.GAUGE_INT64 && type != Type.CUMULATIVE_INT64) { + return type != Type.GAUGE_DISTRIBUTION && type != Type.CUMULATIVE_DISTRIBUTION + ? com.google.api.MetricDescriptor.ValueType.UNRECOGNIZED + : com.google.api.MetricDescriptor.ValueType.DISTRIBUTION; + } else { + return com.google.api.MetricDescriptor.ValueType.INT64; + } + } else { + return com.google.api.MetricDescriptor.ValueType.DOUBLE; + } + } + + static TimeSeries convertTimeSeries( + String metricName, + io.opencensus.metrics.export.MetricDescriptor.Type metricType, + List labelKeys, + List labelValues, + io.opencensus.metrics.export.TimeSeries timeSeries, + MonitoredResource monitoredResource, + String domain, + String projectId) { + + TimeSeries.Builder builder = TimeSeries.newBuilder(); + builder.setMetricKind(createMetricKind(metricType)); + builder.setResource(monitoredResource); + builder.setValueType(createValueType(metricType)); + builder.setMetric(createMetric(metricName, labelKeys, labelValues, domain)); + io.opencensus.common.Timestamp startTimeStamp = timeSeries.getStartTimestamp(); + for (io.opencensus.metrics.export.Point point : timeSeries.getPoints()) { + builder.addPoints(createPoint(point, startTimeStamp)); + } + return builder.build(); + } + + @VisibleForTesting + static com.google.api.Metric createMetric( + String metricName, List labelKeys, List labelValues, String domain) { + com.google.api.Metric.Builder builder = com.google.api.Metric.newBuilder(); + builder.setType(generateType(metricName, domain)); + Map stringTagMap = Maps.newHashMap(); + + for (int i = 0; i < labelValues.size(); ++i) { + String value = ((LabelValue) labelValues.get(i)).getValue(); + if (value != null) { + stringTagMap.put(((LabelKey) labelKeys.get(i)).getKey(), value); + } + } + + builder.putAllLabels(stringTagMap); + return builder.build(); + } + + @VisibleForTesting + static com.google.monitoring.v3.Point createPoint( + io.opencensus.metrics.export.Point point, + @Nullable io.opencensus.common.Timestamp startTimestamp) { + com.google.monitoring.v3.TimeInterval.Builder timeIntervalBuilder = TimeInterval.newBuilder(); + timeIntervalBuilder.setEndTime(convertTimestamp(point.getTimestamp())); + if (startTimestamp != null) { + timeIntervalBuilder.setStartTime(convertTimestamp(startTimestamp)); + } + + com.google.monitoring.v3.Point.Builder builder = com.google.monitoring.v3.Point.newBuilder(); + builder.setInterval(timeIntervalBuilder.build()); + builder.setValue(createTypedValue(point.getValue())); + return builder.build(); + } + + @VisibleForTesting + static TypedValue createTypedValue(Value value) { + return (TypedValue) + value.match( + typedValueDoubleFunction, + typedValueLongFunction, + typedValueDistributionFunction, + typedValueSummaryFunction, + Functions.throwIllegalArgumentException()); + } + + @VisibleForTesting + static com.google.api.Distribution createDistribution( + io.opencensus.metrics.export.Distribution distribution) { + com.google.api.Distribution.Builder builder = + com.google.api.Distribution.newBuilder() + .setBucketOptions(createBucketOptions(distribution.getBucketOptions())) + .setCount(distribution.getCount()) + .setMean( + distribution.getCount() == 0L + ? 0.0D + : distribution.getSum() / (double) distribution.getCount()) + .setSumOfSquaredDeviation(distribution.getSumOfSquaredDeviations()); + setBucketCountsAndExemplars(distribution.getBuckets(), builder); + return builder.build(); + } + + @VisibleForTesting + static BucketOptions createBucketOptions( + @Nullable io.opencensus.metrics.export.Distribution.BucketOptions bucketOptions) { + com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder(); + return bucketOptions == null + ? builder.build() + : (BucketOptions) + bucketOptions.match( + bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException()); + } + + static MonitoredResource getDefaultResource() { + com.google.api.MonitoredResource.Builder builder = MonitoredResource.newBuilder(); + builder.setType("bigtable_client_raw"); + return builder.build(); + } + + private static void setBucketCountsAndExemplars( + List buckets, com.google.api.Distribution.Builder builder) { + builder.addBucketCounts(0L); + + for (Bucket bucket : buckets) { + builder.addBucketCounts(bucket.getCount()); + io.opencensus.metrics.data.Exemplar exemplar = bucket.getExemplar(); + if (exemplar != null) { + builder.addExemplars(toProtoExemplar(exemplar)); + } + } + } + + private static com.google.api.Distribution.Exemplar toProtoExemplar( + io.opencensus.metrics.data.Exemplar exemplar) { + com.google.api.Distribution.Exemplar.Builder builder = + com.google.api.Distribution.Exemplar.newBuilder() + .setValue(exemplar.getValue()) + .setTimestamp(convertTimestamp(exemplar.getTimestamp())); + io.opencensus.trace.SpanContext spanContext = null; + + for (Map.Entry attachment : exemplar.getAttachments().entrySet()) { + String key = (String) attachment.getKey(); + AttachmentValue value = (AttachmentValue) attachment.getValue(); + if ("SpanContext".equals(key)) { + spanContext = ((AttachmentValueSpanContext) value).getSpanContext(); + } else { + builder.addAttachments(toProtoStringAttachment(value)); + } + } + + return builder.build(); + } + + private static Any toProtoStringAttachment(AttachmentValue attachmentValue) { + return Any.newBuilder() + .setTypeUrl("type.googleapis.com/google.protobuf.StringValue") + .setValue(ByteString.copyFromUtf8(attachmentValue.getValue())) + .build(); + } + + private static Any toProtoSpanContextAttachment( + com.google.monitoring.v3.SpanContext protoSpanContext) { + return Any.newBuilder() + .setTypeUrl("type.googleapis.com/google.monitoring.v3.SpanContext") + .setValue(protoSpanContext.toByteString()) + .build(); + } + + private static com.google.monitoring.v3.SpanContext toProtoSpanContext( + io.opencensus.trace.SpanContext spanContext, String projectId) { + String spanName = + String.format( + "projects/%s/traces/%s/spans/%s", + projectId, + spanContext.getTraceId().toLowerBase16(), + spanContext.getSpanId().toLowerBase16()); + return com.google.monitoring.v3.SpanContext.newBuilder().setSpanName(spanName).build(); + } + + @VisibleForTesting + static com.google.protobuf.Timestamp convertTimestamp( + io.opencensus.common.Timestamp censusTimestamp) { + return censusTimestamp.getSeconds() < 0L + ? com.google.protobuf.Timestamp.newBuilder().build() + : com.google.protobuf.Timestamp.newBuilder() + .setSeconds(censusTimestamp.getSeconds()) + .setNanos(censusTimestamp.getNanos()) + .build(); + } + + private BigtableStackdriverExportUtils() {} + + static { + logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); + typedValueDoubleFunction = + new io.opencensus.common.Function() { + public TypedValue apply(Double arg) { + com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); + builder.setDoubleValue(arg); + return builder.build(); + } + }; + typedValueLongFunction = + new io.opencensus.common.Function() { + public TypedValue apply(Long arg) { + com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); + builder.setInt64Value(arg); + return builder.build(); + } + }; + typedValueDistributionFunction = + new io.opencensus.common.Function() { + public TypedValue apply(io.opencensus.metrics.export.Distribution arg) { + com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); + return builder + .setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg)) + .build(); + } + }; + typedValueSummaryFunction = + new io.opencensus.common.Function() { + public TypedValue apply(Summary arg) { + com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); + return builder.build(); + } + }; + bucketOptionsExplicitFunction = + new Function() { + public BucketOptions apply(ExplicitOptions arg) { + com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder(); + com.google.api.Distribution.BucketOptions.Explicit.Builder explicitBuilder = + Explicit.newBuilder(); + explicitBuilder.addBounds(0.0D); + explicitBuilder.addAllBounds(arg.getBucketBoundaries()); + builder.setExplicitBuckets(explicitBuilder.build()); + return builder.build(); + } + }; + } +} diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java new file mode 100644 index 000000000000..e9f84bc76045 --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java @@ -0,0 +1,113 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats.exporter; + +import com.google.api.MonitoredResource; +import com.google.api.core.InternalApi; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.auth.Credentials; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.MetricServiceSettings; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.opencensus.common.Duration; +import io.opencensus.exporter.metrics.util.IntervalMetricReader; +import io.opencensus.exporter.metrics.util.MetricReader; +import io.opencensus.metrics.Metrics; +import java.io.IOException; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +@ThreadSafe +@InternalApi +public class BigtableStackdriverStatsExporter { + @VisibleForTesting static final Object monitor = new Object(); + + @Nullable + @GuardedBy("monitor") + private static BigtableStackdriverStatsExporter instance = null; + + private static final String EXPORTER_SPAN_NAME = "ExportMetricsToStackdriver"; + // private static final String USER_AGENT_KEY = "user-agent"; + // private static final String USER_AGENT = "bigtable-java" + Version.VERSION; + // private static final HeaderProvider OPENCENSUS_USER_AGENT_HEADER_PROVIDER = + // FixedHeaderProvider.create(new String[] {USER_AGENT_KEY, USER_AGENT}); + private final IntervalMetricReader intervalMetricReader; + + private BigtableStackdriverStatsExporter( + String projectId, + MetricServiceClient metricServiceClient, + Duration exportInterval, + MonitoredResource monitoredResource) { + IntervalMetricReader.Options.Builder intervalMetricReaderOptionsBuilder = + IntervalMetricReader.Options.builder(); + intervalMetricReaderOptionsBuilder.setExportInterval(exportInterval); + this.intervalMetricReader = + IntervalMetricReader.create( + new BigtableCreateTimeSeriesExporter(projectId, metricServiceClient, monitoredResource), + MetricReader.create( + io.opencensus.exporter.metrics.util.MetricReader.Options.builder() + .setMetricProducerManager( + Metrics.getExportComponent().getMetricProducerManager()) + .setSpanName(EXPORTER_SPAN_NAME) + .build()), + intervalMetricReaderOptionsBuilder.build()); + } + + public static void register(@Nullable Credentials credentials, String projectId) throws IOException { + synchronized (monitor) { + Preconditions.checkState(instance == null, "Stackdriver stats exporter is already created"); + MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); + instance = new BigtableStackdriverStatsExporter(projectId, client, Duration.create(600, 0), + BigtableStackdriverExportUtils.getDefaultResource()); + } + } + + @GuardedBy("monitor") + @VisibleForTesting + static MetricServiceClient createMetricServiceClient( + @Nullable Credentials credentials, Duration deadline) throws IOException { + com.google.cloud.monitoring.v3.MetricServiceSettings.Builder settingsBuilder = + (com.google.cloud.monitoring.v3.MetricServiceSettings.Builder) + MetricServiceSettings.newBuilder() + .setTransportChannelProvider( + InstantiatingGrpcChannelProvider.newBuilder() + // + // .setHeaderProvider(OPENCENSUS_USER_AGENT_HEADER_PROVIDER) + .build()); + if (credentials != null) { + settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); + } + + org.threeten.bp.Duration stackdriverDuration = + org.threeten.bp.Duration.ofMillis(deadline.toMillis()); + settingsBuilder.createMetricDescriptorSettings().setSimpleTimeoutNoRetries(stackdriverDuration); + settingsBuilder.createTimeSeriesSettings().setSimpleTimeoutNoRetries(stackdriverDuration); + return MetricServiceClient.create(settingsBuilder.build()); + } + + public static void unregister() { + synchronized (monitor) { + if (instance != null) { + instance.intervalMetricReader.stop(); + } + + instance = null; + } + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java new file mode 100644 index 000000000000..47d1e9acf8c6 --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats.exporter; + + import static com.google.common.truth.Truth.assertThat; + import static org.mockito.Mockito.mock; + import static org.mockito.Mockito.when; + + import com.google.api.MonitoredResource; + import com.google.api.gax.rpc.UnaryCallable; + import com.google.cloud.bigtable.stats.BuiltinMeasureConstants; + import io.opencensus.common.Timestamp; + import io.opencensus.metrics.LabelKey; + import io.opencensus.metrics.LabelValue; + import io.opencensus.metrics.export.Metric; + import io.opencensus.metrics.export.MetricDescriptor; + import io.opencensus.metrics.export.Point; + import io.opencensus.metrics.export.Value; + import com.google.cloud.monitoring.v3.MetricServiceClient; + import com.google.cloud.monitoring.v3.stub.MetricServiceStub; + import com.google.monitoring.v3.CreateTimeSeriesRequest; + import com.google.protobuf.Empty; + import java.util.Arrays; + import org.junit.After; + import org.junit.Before; + import org.junit.Rule; + import org.junit.Test; + import org.mockito.ArgumentCaptor; + import org.mockito.Mock; + import org.mockito.junit.MockitoJUnit; + import org.mockito.junit.MockitoRule; + +public class BigtableCreateTimeSeriesExporterTest { + + private static final String projectId = "fake-project"; + private static final String instanceId = "fake-instance"; + private static final String appProfileId = "default"; + private static final String tableId = "fake-table"; + private static final String zone = "us-east-1"; + private static final String cluster = "cluster-1"; + + @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock private MetricServiceStub mockMetricServiceStub; + private MetricServiceClient fakeMetricServiceClient; + private BigtableCreateTimeSeriesExporter exporter; + + @Before + public void setUp() { + + fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); + + exporter = + new BigtableCreateTimeSeriesExporter( + projectId, + fakeMetricServiceClient, + MonitoredResource.newBuilder().setType("bigtable-table").build()); + } + + @After + public void tearDown() {} + + @Test + public void testTimeSeries() { + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); + + UnaryCallable mockCallable = mock(UnaryCallable.class); + when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); + when(mockCallable.call(argumentCaptor.capture())).thenReturn(Empty.getDefaultInstance()); + + double fakeValue = 10.0; + Metric fakeMetric = + Metric.create( + MetricDescriptor.create( + "bigtable/test", + "descritpion", + "ms", + MetricDescriptor.Type.CUMULATIVE_DOUBLE, + Arrays.asList( + LabelKey.create(BuiltinMeasureConstants.PROJECT_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.INSTANCE_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.TABLE.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.CLUSTER.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.ZONE.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.APP_PROFILE.getName(), ""))), + Arrays.asList( + io.opencensus.metrics.export.TimeSeries + .create( + Arrays.asList( + LabelValue.create(projectId), + LabelValue.create(instanceId), + LabelValue.create(tableId), + LabelValue.create(cluster), + LabelValue.create(zone), + LabelValue.create(appProfileId)), + Arrays.asList( + Point.create( + Value.doubleValue(fakeValue), + Timestamp.fromMillis(System.currentTimeMillis()))), + Timestamp.fromMillis(System.currentTimeMillis())))); + + exporter.export(Arrays.asList(fakeMetric)); + + CreateTimeSeriesRequest request = argumentCaptor.getValue(); + + assertThat(request.getTimeSeriesList()).hasSize(1); + + com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); + + assertThat(timeSeries.getResource().getLabelsMap()) + .containsExactly( + BuiltinMeasureConstants.PROJECT_ID.getName(), projectId, + BuiltinMeasureConstants.INSTANCE_ID.getName(), instanceId, + BuiltinMeasureConstants.TABLE.getName(), tableId, + BuiltinMeasureConstants.CLUSTER.getName(), cluster, + BuiltinMeasureConstants.ZONE.getName(), zone); + + assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(2); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId); + assertThat(timeSeries.getMetric().getLabelsMap()).containsKey(BuiltinMeasureConstants.CLIENT_ID.getName()); + + assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); + } + + private class FakeMetricServiceClient extends MetricServiceClient { + + protected FakeMetricServiceClient(MetricServiceStub stub) { + super(stub); + } + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 0ee0d8b24a4c..e6016b0bb2ea 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -20,6 +20,7 @@ import com.google.api.gax.batching.Batcher; import com.google.api.gax.batching.FlowController; import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.UnaryCallSettings; @@ -27,10 +28,14 @@ import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; +import com.google.cloud.bigtable.stats.exporter.BigtableStackdriverStatsExporter; import com.google.common.base.MoreObjects; import com.google.common.base.Strings; import io.grpc.ManagedChannelBuilder; + +import java.io.IOException; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -70,6 +75,7 @@ public final class BigtableDataSettings { private static final Logger LOGGER = Logger.getLogger(BigtableDataSettings.class.getName()); private static final String BIGTABLE_EMULATOR_HOST_ENV_VAR = "BIGTABLE_EMULATOR_HOST"; + private static final AtomicBoolean BUILTIN_METRICS_REGISTERED = new AtomicBoolean(false); private final EnhancedBigtableStubSettings stubSettings; @@ -242,6 +248,15 @@ public Long getBatchMutationsTargetRpcLatencyMs() { return stubSettings.bulkMutateRowsSettings().getTargetRpcLatencyMs(); } + /** + * Gets if built in metrics are registered. + * @return + */ + @BetaApi("Built in metrics is not currently stable and may change in the future") + public boolean isBuiltinMetricsRegistered() { + return BUILTIN_METRICS_REGISTERED.get(); + } + /** Returns the underlying RPC settings. */ public EnhancedBigtableStubSettings getStubSettings() { return stubSettings; @@ -463,6 +478,32 @@ public Long getTargetRpcLatencyMsForBatchMutation() { return stubSettings.bulkMutateRowsSettings().getTargetRpcLatencyMs(); } + /** Register built in metrics. **/ + @BetaApi("Built in metric is not currently stable and may change in the future") + public Builder registerBuiltinMetrics() throws IOException { + if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { + BigtableStackdriverStatsExporter.register( + stubSettings.getCredentialsProvider().getCredentials(), + stubSettings.getProjectId()); + } + return this; + } + + /** Unregister built in metrics. **/ + @BetaApi("Built in metrics is not currently stable and may change in the future") + public Builder unregisterBuiltinMetrics() { + if (BUILTIN_METRICS_REGISTERED.compareAndSet(true, false)) { + BigtableStackdriverStatsExporter.unregister(); + } + return this; + } + + /** Gets if built in metrics are registered */ + @BetaApi("Built in metric is not currently stable and may change in the future") + public boolean isBuiltinMetricsRegistered() { + return BUILTIN_METRICS_REGISTERED.get(); + } + /** * Returns the underlying settings for making RPC calls. The settings should be changed with * care. From 07615bd8d85a5d917874e37a91942062b6e14c0c Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 11 Jul 2022 13:57:04 -0400 Subject: [PATCH 06/30] fix tests --- google-cloud-bigtable-stats/pom.xml | 6 +- .../BigtableCreateTimeSeriesExporter.java | 3 +- .../BigtableStackdriverExportUtils.java | 5 +- .../BigtableStackdriverStatsExporter.java | 11 +- .../BigtableCreateTimeSeriesExporterTest.java | 147 ++++++++++++++++++ .../stats/StatsRecorderWrapperTest.java | 1 + .../BigtableCreateTimeSeriesExporterTest.java | 146 ----------------- .../data/v2/BigtableDataSettings.java | 12 +- pom.xml | 11 +- 9 files changed, 165 insertions(+), 177 deletions(-) rename google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/{exporter => }/BigtableCreateTimeSeriesExporter.java (97%) rename google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/{exporter => }/BigtableStackdriverExportUtils.java (98%) rename google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/{exporter => }/BigtableStackdriverStatsExporter.java (95%) create mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java delete mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 23e20d4b2488..5ded05db56ae 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -51,16 +51,12 @@ com.google.cloud google-cloud-monitoring - 3.2.9 + 3.3.1 io.opencensus opencensus-api - - io.opencensus - opencensus-impl - io.opencensus opencensus-exporter-stats-stackdriver diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java similarity index 97% rename from google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java rename to google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index e66c86613910..268ae03348c6 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.stats.exporter; +package com.google.cloud.bigtable.stats; import com.google.api.MonitoredResource; import com.google.api.gax.rpc.ApiException; -import com.google.cloud.bigtable.stats.BuiltinMeasureConstants; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.ProjectName; diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java similarity index 98% rename from google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java rename to google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index f0fc1118ad85..ba4e49d6ea85 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.stats.exporter; +package com.google.cloud.bigtable.stats; import com.google.api.Distribution.BucketOptions; import com.google.api.Distribution.BucketOptions.Explicit; @@ -29,7 +29,6 @@ import io.opencensus.common.Function; import io.opencensus.common.Functions; import io.opencensus.contrib.exemplar.util.AttachmentValueSpanContext; -import io.opencensus.contrib.resource.util.ResourceUtils; import io.opencensus.metrics.LabelKey; import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.data.AttachmentValue; @@ -38,8 +37,6 @@ import io.opencensus.metrics.export.MetricDescriptor.Type; import io.opencensus.metrics.export.Summary; import io.opencensus.metrics.export.Value; -import io.opencensus.resource.Resource; - import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.UnknownHostException; diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java similarity index 95% rename from google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java rename to google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index e9f84bc76045..c9c347486672 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/exporter/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.stats.exporter; +package com.google.cloud.bigtable.stats; import com.google.api.MonitoredResource; import com.google.api.core.InternalApi; @@ -69,11 +69,16 @@ private BigtableStackdriverStatsExporter( intervalMetricReaderOptionsBuilder.build()); } - public static void register(@Nullable Credentials credentials, String projectId) throws IOException { + public static void register(@Nullable Credentials credentials, String projectId) + throws IOException { synchronized (monitor) { Preconditions.checkState(instance == null, "Stackdriver stats exporter is already created"); MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); - instance = new BigtableStackdriverStatsExporter(projectId, client, Duration.create(600, 0), + instance = + new BigtableStackdriverStatsExporter( + projectId, + client, + Duration.create(600, 0), BigtableStackdriverExportUtils.getDefaultResource()); } } diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java new file mode 100644 index 000000000000..931325591a45 --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.api.MonitoredResource; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.cloud.bigtable.stats.BigtableCreateTimeSeriesExporter; +import com.google.cloud.bigtable.stats.BuiltinMeasureConstants; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.stub.MetricServiceStub; +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.protobuf.Empty; +import io.opencensus.common.Timestamp; +import io.opencensus.metrics.LabelKey; +import io.opencensus.metrics.LabelValue; +import io.opencensus.metrics.export.Metric; +import io.opencensus.metrics.export.MetricDescriptor; +import io.opencensus.metrics.export.Point; +import io.opencensus.metrics.export.Value; +import java.util.Arrays; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigtableCreateTimeSeriesExporterTest { + + private static final String projectId = "fake-project"; + private static final String instanceId = "fake-instance"; + private static final String appProfileId = "default"; + private static final String tableId = "fake-table"; + private static final String zone = "us-east-1"; + private static final String cluster = "cluster-1"; + + @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock private MetricServiceStub mockMetricServiceStub; + private MetricServiceClient fakeMetricServiceClient; + private BigtableCreateTimeSeriesExporter exporter; + + @Before + public void setUp() { + + fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); + + exporter = + new BigtableCreateTimeSeriesExporter( + projectId, + fakeMetricServiceClient, + MonitoredResource.newBuilder().setType("bigtable-table").build()); + } + + @After + public void tearDown() {} + + @Test + public void testTimeSeries() { + ArgumentCaptor argumentCaptor = + ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); + + UnaryCallable mockCallable = mock(UnaryCallable.class); + when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); + when(mockCallable.call(argumentCaptor.capture())).thenReturn(Empty.getDefaultInstance()); + + double fakeValue = 10.0; + Metric fakeMetric = + Metric.create( + MetricDescriptor.create( + "bigtable/test", + "descritpion", + "ms", + MetricDescriptor.Type.CUMULATIVE_DOUBLE, + Arrays.asList( + LabelKey.create(BuiltinMeasureConstants.PROJECT_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.INSTANCE_ID.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.TABLE.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.CLUSTER.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.ZONE.getName(), ""), + LabelKey.create(BuiltinMeasureConstants.APP_PROFILE.getName(), ""))), + Arrays.asList( + io.opencensus.metrics.export.TimeSeries.create( + Arrays.asList( + LabelValue.create(projectId), + LabelValue.create(instanceId), + LabelValue.create(tableId), + LabelValue.create(cluster), + LabelValue.create(zone), + LabelValue.create(appProfileId)), + Arrays.asList( + Point.create( + Value.doubleValue(fakeValue), + Timestamp.fromMillis(System.currentTimeMillis()))), + Timestamp.fromMillis(System.currentTimeMillis())))); + + exporter.export(Arrays.asList(fakeMetric)); + + CreateTimeSeriesRequest request = argumentCaptor.getValue(); + + assertThat(request.getTimeSeriesList()).hasSize(1); + + com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); + + assertThat(timeSeries.getResource().getLabelsMap()) + .containsExactly( + BuiltinMeasureConstants.PROJECT_ID.getName(), projectId, + BuiltinMeasureConstants.INSTANCE_ID.getName(), instanceId, + BuiltinMeasureConstants.TABLE.getName(), tableId, + BuiltinMeasureConstants.CLUSTER.getName(), cluster, + BuiltinMeasureConstants.ZONE.getName(), zone); + + assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(2); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId); + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsKey(BuiltinMeasureConstants.CLIENT_ID.getName()); + + assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); + } + + private class FakeMetricServiceClient extends MetricServiceClient { + + protected FakeMetricServiceClient(MetricServiceStub stub) { + super(stub); + } + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java index 11e9184e10e9..abf00e71b340 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java @@ -92,6 +92,7 @@ public void testStreamingOperation() throws InterruptedException { recorderWrapper.putGfeMissingHeaders(connectivityErrorCount); recorderWrapper.putFirstResponseLatencies(firstResponseLatency); recorderWrapper.putBatchRequestThrottled(throttlingLatency); + recorderWrapper.record("OK", TABLE_ID, ZONE, CLUSTER); Thread.sleep(100); diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java deleted file mode 100644 index 47d1e9acf8c6..000000000000 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/exporter/BigtableCreateTimeSeriesExporterTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats.exporter; - - import static com.google.common.truth.Truth.assertThat; - import static org.mockito.Mockito.mock; - import static org.mockito.Mockito.when; - - import com.google.api.MonitoredResource; - import com.google.api.gax.rpc.UnaryCallable; - import com.google.cloud.bigtable.stats.BuiltinMeasureConstants; - import io.opencensus.common.Timestamp; - import io.opencensus.metrics.LabelKey; - import io.opencensus.metrics.LabelValue; - import io.opencensus.metrics.export.Metric; - import io.opencensus.metrics.export.MetricDescriptor; - import io.opencensus.metrics.export.Point; - import io.opencensus.metrics.export.Value; - import com.google.cloud.monitoring.v3.MetricServiceClient; - import com.google.cloud.monitoring.v3.stub.MetricServiceStub; - import com.google.monitoring.v3.CreateTimeSeriesRequest; - import com.google.protobuf.Empty; - import java.util.Arrays; - import org.junit.After; - import org.junit.Before; - import org.junit.Rule; - import org.junit.Test; - import org.mockito.ArgumentCaptor; - import org.mockito.Mock; - import org.mockito.junit.MockitoJUnit; - import org.mockito.junit.MockitoRule; - -public class BigtableCreateTimeSeriesExporterTest { - - private static final String projectId = "fake-project"; - private static final String instanceId = "fake-instance"; - private static final String appProfileId = "default"; - private static final String tableId = "fake-table"; - private static final String zone = "us-east-1"; - private static final String cluster = "cluster-1"; - - @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); - - @Mock private MetricServiceStub mockMetricServiceStub; - private MetricServiceClient fakeMetricServiceClient; - private BigtableCreateTimeSeriesExporter exporter; - - @Before - public void setUp() { - - fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); - - exporter = - new BigtableCreateTimeSeriesExporter( - projectId, - fakeMetricServiceClient, - MonitoredResource.newBuilder().setType("bigtable-table").build()); - } - - @After - public void tearDown() {} - - @Test - public void testTimeSeries() { - ArgumentCaptor argumentCaptor = - ArgumentCaptor.forClass(CreateTimeSeriesRequest.class); - - UnaryCallable mockCallable = mock(UnaryCallable.class); - when(mockMetricServiceStub.createServiceTimeSeriesCallable()).thenReturn(mockCallable); - when(mockCallable.call(argumentCaptor.capture())).thenReturn(Empty.getDefaultInstance()); - - double fakeValue = 10.0; - Metric fakeMetric = - Metric.create( - MetricDescriptor.create( - "bigtable/test", - "descritpion", - "ms", - MetricDescriptor.Type.CUMULATIVE_DOUBLE, - Arrays.asList( - LabelKey.create(BuiltinMeasureConstants.PROJECT_ID.getName(), ""), - LabelKey.create(BuiltinMeasureConstants.INSTANCE_ID.getName(), ""), - LabelKey.create(BuiltinMeasureConstants.TABLE.getName(), ""), - LabelKey.create(BuiltinMeasureConstants.CLUSTER.getName(), ""), - LabelKey.create(BuiltinMeasureConstants.ZONE.getName(), ""), - LabelKey.create(BuiltinMeasureConstants.APP_PROFILE.getName(), ""))), - Arrays.asList( - io.opencensus.metrics.export.TimeSeries - .create( - Arrays.asList( - LabelValue.create(projectId), - LabelValue.create(instanceId), - LabelValue.create(tableId), - LabelValue.create(cluster), - LabelValue.create(zone), - LabelValue.create(appProfileId)), - Arrays.asList( - Point.create( - Value.doubleValue(fakeValue), - Timestamp.fromMillis(System.currentTimeMillis()))), - Timestamp.fromMillis(System.currentTimeMillis())))); - - exporter.export(Arrays.asList(fakeMetric)); - - CreateTimeSeriesRequest request = argumentCaptor.getValue(); - - assertThat(request.getTimeSeriesList()).hasSize(1); - - com.google.monitoring.v3.TimeSeries timeSeries = request.getTimeSeriesList().get(0); - - assertThat(timeSeries.getResource().getLabelsMap()) - .containsExactly( - BuiltinMeasureConstants.PROJECT_ID.getName(), projectId, - BuiltinMeasureConstants.INSTANCE_ID.getName(), instanceId, - BuiltinMeasureConstants.TABLE.getName(), tableId, - BuiltinMeasureConstants.CLUSTER.getName(), cluster, - BuiltinMeasureConstants.ZONE.getName(), zone); - - assertThat(timeSeries.getMetric().getLabelsMap()).hasSize(2); - assertThat(timeSeries.getMetric().getLabelsMap()) - .containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId); - assertThat(timeSeries.getMetric().getLabelsMap()).containsKey(BuiltinMeasureConstants.CLIENT_ID.getName()); - - assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); - } - - private class FakeMetricServiceClient extends MetricServiceClient { - - protected FakeMetricServiceClient(MetricServiceStub stub) { - super(stub); - } - } -} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index e6016b0bb2ea..81a15950904b 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -20,7 +20,6 @@ import com.google.api.gax.batching.Batcher; import com.google.api.gax.batching.FlowController; import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.UnaryCallSettings; @@ -28,11 +27,10 @@ import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; -import com.google.cloud.bigtable.stats.exporter.BigtableStackdriverStatsExporter; +import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter; import com.google.common.base.MoreObjects; import com.google.common.base.Strings; import io.grpc.ManagedChannelBuilder; - import java.io.IOException; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -250,6 +248,7 @@ public Long getBatchMutationsTargetRpcLatencyMs() { /** * Gets if built in metrics are registered. + * * @return */ @BetaApi("Built in metrics is not currently stable and may change in the future") @@ -478,18 +477,17 @@ public Long getTargetRpcLatencyMsForBatchMutation() { return stubSettings.bulkMutateRowsSettings().getTargetRpcLatencyMs(); } - /** Register built in metrics. **/ + /** Register built in metrics. * */ @BetaApi("Built in metric is not currently stable and may change in the future") public Builder registerBuiltinMetrics() throws IOException { if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { BigtableStackdriverStatsExporter.register( - stubSettings.getCredentialsProvider().getCredentials(), - stubSettings.getProjectId()); + stubSettings.getCredentialsProvider().getCredentials(), stubSettings.getProjectId()); } return this; } - /** Unregister built in metrics. **/ + /** Unregister built in metrics. * */ @BetaApi("Built in metrics is not currently stable and may change in the future") public Builder unregisterBuiltinMetrics() { if (BUILTIN_METRICS_REGISTERED.compareAndSet(true, false)) { diff --git a/pom.xml b/pom.xml index 908ac282f333..11db087d5ef8 100644 --- a/pom.xml +++ b/pom.xml @@ -212,15 +212,6 @@ - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.4 - - - @@ -352,11 +343,11 @@ google-cloud-bigtable - google-cloud-bigtable-stats grpc-google-cloud-bigtable-admin-v2 grpc-google-cloud-bigtable-v2 proto-google-cloud-bigtable-admin-v2 proto-google-cloud-bigtable-v2 + google-cloud-bigtable-stats google-cloud-bigtable-emulator-core google-cloud-bigtable-emulator google-cloud-bigtable-bom From f499a1b31f6fa25a50a4a748c271f73d4ac70d39 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 11 Jul 2022 14:58:00 -0400 Subject: [PATCH 07/30] fix dependency --- google-cloud-bigtable-stats/pom.xml | 53 ++++++-- .../BigtableCreateTimeSeriesExporter.java | 10 +- .../stats/BigtableStackdriverExportUtils.java | 116 ++++++------------ .../BigtableStackdriverStatsExporter.java | 25 ++-- .../BigtableCreateTimeSeriesExporterTest.java | 2 - google-cloud-bigtable/pom.xml | 8 ++ .../data/v2/stub/EnhancedBigtableStub.java | 2 + 7 files changed, 106 insertions(+), 110 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 5ded05db56ae..905475c97810 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -25,6 +25,21 @@ pom import + + io.opencensus + opencensus-exporter-metrics-util + 0.31.1 + + + com.google.cloud + google-cloud-monitoring + 3.3.1 + + + com.google.api.grpc + proto-google-cloud-monitoring-v3 + 3.3.1 + @@ -51,32 +66,52 @@ com.google.cloud google-cloud-monitoring - 3.3.1 io.opencensus opencensus-api + + com.google.auth + google-auth-library-credentials + + + com.google.api.grpc + proto-google-cloud-monitoring-v3 + + + com.google.api + gax-grpc + + + org.threeten + threetenbp + + + com.google.protobuf + protobuf-java + io.opencensus - opencensus-exporter-stats-stackdriver - - - com.google.cloud - google-cloud-monitoring - - + opencensus-exporter-metrics-util com.google.guava guava - + + com.google.api.grpc + proto-google-common-protos + io.opencensus opencensus-impl runtime + + com.google.code.findbugs + jsr305 + com.google.truth truth diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index 268ae03348c6..eec415f46cc4 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,10 +34,12 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); + + private static final String DOMAIN = "bigtable.googleapis.com/internal/client/"; + private final ProjectName projectName; private final MetricServiceClient metricServiceClient; private final MonitoredResource monitoredResource; - private final String domain; BigtableCreateTimeSeriesExporter( String projectId, @@ -46,7 +48,6 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { this.projectName = ProjectName.newBuilder().setProject(projectId).build(); this.metricServiceClient = metricServiceClient; this.monitoredResource = monitoredResource; - this.domain = "bigtable.googleapis.com/client/"; } public void export(Collection metrics) { @@ -101,8 +102,7 @@ public void export(Collection metrics) { updatedValues, timeSeries, monitoredResourceBuilder.build(), - this.domain, - this.projectName.getProject())); + DOMAIN)); } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index ba4e49d6ea85..d45f26d52ac1 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,9 +26,7 @@ import com.google.monitoring.v3.TypedValue; import com.google.protobuf.Any; import com.google.protobuf.ByteString; -import io.opencensus.common.Function; import io.opencensus.common.Functions; -import io.opencensus.contrib.exemplar.util.AttachmentValueSpanContext; import io.opencensus.metrics.LabelKey; import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.data.AttachmentValue; @@ -115,8 +113,7 @@ static TimeSeries convertTimeSeries( List labelValues, io.opencensus.metrics.export.TimeSeries timeSeries, MonitoredResource monitoredResource, - String domain, - String projectId) { + String domain) { TimeSeries.Builder builder = TimeSeries.newBuilder(); builder.setMetricKind(createMetricKind(metricType)); @@ -138,9 +135,9 @@ static com.google.api.Metric createMetric( Map stringTagMap = Maps.newHashMap(); for (int i = 0; i < labelValues.size(); ++i) { - String value = ((LabelValue) labelValues.get(i)).getValue(); + String value = labelValues.get(i).getValue(); if (value != null) { - stringTagMap.put(((LabelKey) labelKeys.get(i)).getKey(), value); + stringTagMap.put(labelKeys.get(i).getKey(), value); } } @@ -166,13 +163,12 @@ static com.google.monitoring.v3.Point createPoint( @VisibleForTesting static TypedValue createTypedValue(Value value) { - return (TypedValue) - value.match( - typedValueDoubleFunction, - typedValueLongFunction, - typedValueDistributionFunction, - typedValueSummaryFunction, - Functions.throwIllegalArgumentException()); + return value.match( + typedValueDoubleFunction, + typedValueLongFunction, + typedValueDistributionFunction, + typedValueSummaryFunction, + Functions.throwIllegalArgumentException()); } @VisibleForTesting @@ -197,9 +193,8 @@ static BucketOptions createBucketOptions( com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder(); return bucketOptions == null ? builder.build() - : (BucketOptions) - bucketOptions.match( - bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException()); + : bucketOptions.match( + bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException()); } static MonitoredResource getDefaultResource() { @@ -227,16 +222,11 @@ private static com.google.api.Distribution.Exemplar toProtoExemplar( com.google.api.Distribution.Exemplar.newBuilder() .setValue(exemplar.getValue()) .setTimestamp(convertTimestamp(exemplar.getTimestamp())); - io.opencensus.trace.SpanContext spanContext = null; for (Map.Entry attachment : exemplar.getAttachments().entrySet()) { - String key = (String) attachment.getKey(); - AttachmentValue value = (AttachmentValue) attachment.getValue(); - if ("SpanContext".equals(key)) { - spanContext = ((AttachmentValueSpanContext) value).getSpanContext(); - } else { - builder.addAttachments(toProtoStringAttachment(value)); - } + String key = attachment.getKey(); + AttachmentValue value = attachment.getValue(); + builder.addAttachments(toProtoStringAttachment(value)); } return builder.build(); @@ -249,25 +239,6 @@ private static Any toProtoStringAttachment(AttachmentValue attachmentValue) { .build(); } - private static Any toProtoSpanContextAttachment( - com.google.monitoring.v3.SpanContext protoSpanContext) { - return Any.newBuilder() - .setTypeUrl("type.googleapis.com/google.monitoring.v3.SpanContext") - .setValue(protoSpanContext.toByteString()) - .build(); - } - - private static com.google.monitoring.v3.SpanContext toProtoSpanContext( - io.opencensus.trace.SpanContext spanContext, String projectId) { - String spanName = - String.format( - "projects/%s/traces/%s/spans/%s", - projectId, - spanContext.getTraceId().toLowerBase16(), - spanContext.getSpanId().toLowerBase16()); - return com.google.monitoring.v3.SpanContext.newBuilder().setSpanName(spanName).build(); - } - @VisibleForTesting static com.google.protobuf.Timestamp convertTimestamp( io.opencensus.common.Timestamp censusTimestamp) { @@ -284,48 +255,37 @@ private BigtableStackdriverExportUtils() {} static { logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); typedValueDoubleFunction = - new io.opencensus.common.Function() { - public TypedValue apply(Double arg) { - com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); - builder.setDoubleValue(arg); - return builder.build(); - } + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + builder.setDoubleValue(arg); + return builder.build(); }; typedValueLongFunction = - new io.opencensus.common.Function() { - public TypedValue apply(Long arg) { - com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); - builder.setInt64Value(arg); - return builder.build(); - } + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + builder.setInt64Value(arg); + return builder.build(); }; typedValueDistributionFunction = - new io.opencensus.common.Function() { - public TypedValue apply(io.opencensus.metrics.export.Distribution arg) { - com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); - return builder - .setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg)) - .build(); - } + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + return builder + .setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg)) + .build(); }; typedValueSummaryFunction = - new io.opencensus.common.Function() { - public TypedValue apply(Summary arg) { - com.google.monitoring.v3.TypedValue.Builder builder = TypedValue.newBuilder(); - return builder.build(); - } + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + return builder.build(); }; bucketOptionsExplicitFunction = - new Function() { - public BucketOptions apply(ExplicitOptions arg) { - com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder(); - com.google.api.Distribution.BucketOptions.Explicit.Builder explicitBuilder = - Explicit.newBuilder(); - explicitBuilder.addBounds(0.0D); - explicitBuilder.addAllBounds(arg.getBucketBoundaries()); - builder.setExplicitBuckets(explicitBuilder.build()); - return builder.build(); - } + arg -> { + BucketOptions.Builder builder = BucketOptions.newBuilder(); + Explicit.Builder explicitBuilder = Explicit.newBuilder(); + explicitBuilder.addBounds(0.0D); + explicitBuilder.addAllBounds(arg.getBucketBoundaries()); + builder.setExplicitBuckets(explicitBuilder.build()); + return builder.build(); }; } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index c9c347486672..03dfc465e478 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,17 +36,14 @@ @ThreadSafe @InternalApi public class BigtableStackdriverStatsExporter { - @VisibleForTesting static final Object monitor = new Object(); + static final Object monitor = new Object(); @Nullable @GuardedBy("monitor") private static BigtableStackdriverStatsExporter instance = null; - private static final String EXPORTER_SPAN_NAME = "ExportMetricsToStackdriver"; - // private static final String USER_AGENT_KEY = "user-agent"; - // private static final String USER_AGENT = "bigtable-java" + Version.VERSION; - // private static final HeaderProvider OPENCENSUS_USER_AGENT_HEADER_PROVIDER = - // FixedHeaderProvider.create(new String[] {USER_AGENT_KEY, USER_AGENT}); + private static final String EXPORTER_SPAN_NAME = "BigtableExportMetricsToStackdriver"; + private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); private final IntervalMetricReader intervalMetricReader; private BigtableStackdriverStatsExporter( @@ -72,13 +69,14 @@ private BigtableStackdriverStatsExporter( public static void register(@Nullable Credentials credentials, String projectId) throws IOException { synchronized (monitor) { - Preconditions.checkState(instance == null, "Stackdriver stats exporter is already created"); + Preconditions.checkState( + instance == null, "Bigtable Stackdriver stats exporter is already created"); MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); instance = new BigtableStackdriverStatsExporter( projectId, client, - Duration.create(600, 0), + EXPORT_INTERVAL, BigtableStackdriverExportUtils.getDefaultResource()); } } @@ -88,13 +86,8 @@ public static void register(@Nullable Credentials credentials, String projectId) static MetricServiceClient createMetricServiceClient( @Nullable Credentials credentials, Duration deadline) throws IOException { com.google.cloud.monitoring.v3.MetricServiceSettings.Builder settingsBuilder = - (com.google.cloud.monitoring.v3.MetricServiceSettings.Builder) - MetricServiceSettings.newBuilder() - .setTransportChannelProvider( - InstantiatingGrpcChannelProvider.newBuilder() - // - // .setHeaderProvider(OPENCENSUS_USER_AGENT_HEADER_PROVIDER) - .build()); + MetricServiceSettings.newBuilder() + .setTransportChannelProvider(InstantiatingGrpcChannelProvider.newBuilder().build()); if (credentials != null) { settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); } diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 931325591a45..25b5a0b97ea2 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -21,8 +21,6 @@ import com.google.api.MonitoredResource; import com.google.api.gax.rpc.UnaryCallable; -import com.google.cloud.bigtable.stats.BigtableCreateTimeSeriesExporter; -import com.google.cloud.bigtable.stats.BuiltinMeasureConstants; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.cloud.monitoring.v3.stub.MetricServiceStub; import com.google.monitoring.v3.CreateTimeSeriesRequest; diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index e0f7b6f10d92..d2d61565c683 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -60,9 +60,17 @@ + com.google.cloud google-cloud-bigtable-stats + + + + io.opencensus + * + + diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index c6591d588a49..fed6d3cc00a8 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -92,6 +92,7 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; +import com.google.cloud.bigtable.stats.BuiltinViews; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -231,6 +232,7 @@ public static EnhancedBigtableStubSettings finalizeSettings( BuiltinMetricsTracerFactory.create(builtinAttributes), // Add user configured tracer settings.getTracerFactory()))); + BuiltinViews.registerBigtableBuiltinViews(); return builder.build(); } From 3ea37396c8c76399724d4fafb21eca5ac90dc4f5 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 12 Jul 2022 12:52:49 -0400 Subject: [PATCH 08/30] remove unused dependency --- google-cloud-bigtable-stats/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 905475c97810..d5601c4392d0 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -122,11 +122,6 @@ junit test - - io.grpc - grpc-api - test - org.mockito mockito-core From 2032527d80733d7b1599b38a2c40f734c9b2d9c5 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 12 Jul 2022 13:06:31 -0400 Subject: [PATCH 09/30] clean up code --- .../BigtableCreateTimeSeriesExporter.java | 5 +---- .../stats/BigtableStackdriverExportUtils.java | 22 +++++-------------- .../BigtableStackdriverStatsExporter.java | 10 ++++----- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index eec415f46cc4..d22346e48350 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -35,8 +35,6 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); - private static final String DOMAIN = "bigtable.googleapis.com/internal/client/"; - private final ProjectName projectName; private final MetricServiceClient metricServiceClient; private final MonitoredResource monitoredResource; @@ -101,8 +99,7 @@ public void export(Collection metrics) { updatedKeys, updatedValues, timeSeries, - monitoredResourceBuilder.build(), - DOMAIN)); + monitoredResourceBuilder.build())); } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index d45f26d52ac1..25026527bb4f 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -57,6 +57,8 @@ class BigtableStackdriverExportUtils { private static final io.opencensus.common.Function bucketOptionsExplicitFunction; + private static final String DOMAIN = "bigtable.googleapis.com/internal/client/"; + static String generateDefaultTaskValue() { String jvmName = ManagementFactory.getRuntimeMXBean().getName(); if (jvmName.indexOf(64) < 1) { @@ -74,10 +76,6 @@ static String generateDefaultTaskValue() { } } - private static String generateType(String metricName, String domain) { - return domain + metricName; - } - @VisibleForTesting static MetricKind createMetricKind(Type type) { if (type != Type.GAUGE_INT64 && type != Type.GAUGE_DOUBLE) { @@ -112,14 +110,13 @@ static TimeSeries convertTimeSeries( List labelKeys, List labelValues, io.opencensus.metrics.export.TimeSeries timeSeries, - MonitoredResource monitoredResource, - String domain) { + MonitoredResource monitoredResource) { TimeSeries.Builder builder = TimeSeries.newBuilder(); builder.setMetricKind(createMetricKind(metricType)); builder.setResource(monitoredResource); builder.setValueType(createValueType(metricType)); - builder.setMetric(createMetric(metricName, labelKeys, labelValues, domain)); + builder.setMetric(createMetric(metricName, labelKeys, labelValues)); io.opencensus.common.Timestamp startTimeStamp = timeSeries.getStartTimestamp(); for (io.opencensus.metrics.export.Point point : timeSeries.getPoints()) { builder.addPoints(createPoint(point, startTimeStamp)); @@ -129,9 +126,9 @@ static TimeSeries convertTimeSeries( @VisibleForTesting static com.google.api.Metric createMetric( - String metricName, List labelKeys, List labelValues, String domain) { + String metricName, List labelKeys, List labelValues) { com.google.api.Metric.Builder builder = com.google.api.Metric.newBuilder(); - builder.setType(generateType(metricName, domain)); + builder.setType(DOMAIN + metricName); Map stringTagMap = Maps.newHashMap(); for (int i = 0; i < labelValues.size(); ++i) { @@ -197,12 +194,6 @@ static BucketOptions createBucketOptions( bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException()); } - static MonitoredResource getDefaultResource() { - com.google.api.MonitoredResource.Builder builder = MonitoredResource.newBuilder(); - builder.setType("bigtable_client_raw"); - return builder.build(); - } - private static void setBucketCountsAndExemplars( List buckets, com.google.api.Distribution.Builder builder) { builder.addBucketCounts(0L); @@ -224,7 +215,6 @@ private static com.google.api.Distribution.Exemplar toProtoExemplar( .setTimestamp(convertTimestamp(exemplar.getTimestamp())); for (Map.Entry attachment : exemplar.getAttachments().entrySet()) { - String key = attachment.getKey(); AttachmentValue value = attachment.getValue(); builder.addAttachments(toProtoStringAttachment(value)); } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 03dfc465e478..2fd47c372b8c 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -44,6 +44,8 @@ public class BigtableStackdriverStatsExporter { private static final String EXPORTER_SPAN_NAME = "BigtableExportMetricsToStackdriver"; private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); + private static final String RESOURCE_TYPE = "bigtable_client_raw"; + private final IntervalMetricReader intervalMetricReader; private BigtableStackdriverStatsExporter( @@ -72,12 +74,10 @@ public static void register(@Nullable Credentials credentials, String projectId) Preconditions.checkState( instance == null, "Bigtable Stackdriver stats exporter is already created"); MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); + MonitoredResource resourceType = + MonitoredResource.newBuilder().setType(RESOURCE_TYPE).build(); instance = - new BigtableStackdriverStatsExporter( - projectId, - client, - EXPORT_INTERVAL, - BigtableStackdriverExportUtils.getDefaultResource()); + new BigtableStackdriverStatsExporter(projectId, client, EXPORT_INTERVAL, resourceType); } } From fa1756ed652830f62371dc935a977bb762d9dab9 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 14 Jul 2022 17:42:18 -0400 Subject: [PATCH 10/30] udpates on comments --- google-cloud-bigtable-stats/pom.xml | 103 ++++++++++++------ .../BigtableCreateTimeSeriesExporter.java | 47 +------- .../stats/BigtableStackdriverExportUtils.java | 75 +++++++++---- .../BigtableStackdriverStatsExporter.java | 4 - .../BigtableCreateTimeSeriesExporterTest.java | 3 + google-cloud-bigtable/pom.xml | 2 +- .../data/v2/BigtableDataSettings.java | 2 + .../data/v2/stub/EnhancedBigtableStub.java | 2 - 8 files changed, 134 insertions(+), 104 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index d5601c4392d0..b32a9fcbd1db 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -16,6 +16,11 @@ 2.10.2-SNAPSHOT Experimental project to shade OpenCensus dependencies. + + 0.31.1 + 3.3.1 + + @@ -28,22 +33,54 @@ io.opencensus opencensus-exporter-metrics-util - 0.31.1 + ${opencensus.version} com.google.cloud google-cloud-monitoring - 3.3.1 + ${cloud.monitoring.version} com.google.api.grpc proto-google-cloud-monitoring-v3 - 3.3.1 + ${cloud.monitoring.version} + + + io.opencensus + opencensus-api + + + io.opencensus + opencensus-exporter-metrics-util + + + io.opencensus + opencensus-impl + runtime + + + + + com.google.cloud + google-cloud-monitoring + + + com.google.api.grpc + proto-google-cloud-monitoring-v3 + + + com.google.api.grpc + proto-google-common-protos + + + com.google.auth + google-auth-library-credentials + com.google.api gax @@ -63,55 +100,28 @@ com.google.api api-common - - com.google.cloud - google-cloud-monitoring - - - io.opencensus - opencensus-api - - - com.google.auth - google-auth-library-credentials - - - com.google.api.grpc - proto-google-cloud-monitoring-v3 - com.google.api gax-grpc - - org.threeten - threetenbp - com.google.protobuf protobuf-java - - io.opencensus - opencensus-exporter-metrics-util - com.google.guava guava - com.google.api.grpc - proto-google-common-protos - - - io.opencensus - opencensus-impl - runtime + org.threeten + threetenbp com.google.code.findbugs jsr305 + + com.google.truth truth @@ -175,6 +185,31 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-version-consistency + + enforce + + + + + + + + + io.opencensus:*:[${opencensus.version}] + io.opencensus:opencensus-proto:[0.2.0] + + + + + + + diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index d22346e48350..8f118c02eb20 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -16,14 +16,11 @@ package com.google.cloud.bigtable.stats; import com.google.api.MonitoredResource; -import com.google.api.gax.rpc.ApiException; import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.ProjectName; import com.google.monitoring.v3.TimeSeries; import io.opencensus.exporter.metrics.util.MetricExporter; -import io.opencensus.metrics.LabelKey; -import io.opencensus.metrics.LabelValue; import io.opencensus.metrics.export.Metric; import java.util.ArrayList; import java.util.Collection; @@ -38,6 +35,7 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { private final ProjectName projectName; private final MetricServiceClient metricServiceClient; private final MonitoredResource monitoredResource; + private final String clientId; BigtableCreateTimeSeriesExporter( String projectId, @@ -46,6 +44,7 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { this.projectName = ProjectName.newBuilder().setProject(projectId).build(); this.metricServiceClient = metricServiceClient; this.monitoredResource = monitoredResource; + this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue(); } public void export(Collection metrics) { @@ -58,48 +57,14 @@ public void export(Collection metrics) { } for (io.opencensus.metrics.export.TimeSeries timeSeries : metric.getTimeSeriesList()) { - MonitoredResource.Builder monitoredResourceBuilder = this.monitoredResource.toBuilder(); - - List keys = metric.getMetricDescriptor().getLabelKeys(); - List labelValues = timeSeries.getLabelValues(); - - List updatedKeys = new ArrayList<>(); - List updatedValues = new ArrayList<>(); - - for (int i = 0; i < labelValues.size(); i++) { - if (keys.get(i).getKey().equals(BuiltinMeasureConstants.PROJECT_ID.getName())) { - monitoredResourceBuilder.putLabels( - BuiltinMeasureConstants.PROJECT_ID.getName(), labelValues.get(i).getValue()); - } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.INSTANCE_ID.getName())) { - monitoredResourceBuilder.putLabels( - BuiltinMeasureConstants.INSTANCE_ID.getName(), labelValues.get(i).getValue()); - } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.CLUSTER.getName())) { - monitoredResourceBuilder.putLabels( - BuiltinMeasureConstants.CLUSTER.getName(), labelValues.get(i).getValue()); - } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.ZONE.getName())) { - monitoredResourceBuilder.putLabels( - BuiltinMeasureConstants.ZONE.getName(), labelValues.get(i).getValue()); - } else if (keys.get(i).getKey().equals(BuiltinMeasureConstants.TABLE.getName())) { - monitoredResourceBuilder.putLabels( - BuiltinMeasureConstants.TABLE.getName(), labelValues.get(i).getValue()); - } else { - updatedKeys.add(keys.get(i)); - updatedValues.add(labelValues.get(i)); - } - } - - updatedKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_ID.getName(), "client id")); - updatedValues.add( - LabelValue.create(BigtableStackdriverExportUtils.generateDefaultTaskValue())); - timeSeriesList.add( BigtableStackdriverExportUtils.convertTimeSeries( metric.getMetricDescriptor().getName(), metric.getMetricDescriptor().getType(), - updatedKeys, - updatedValues, + metric.getMetricDescriptor().getLabelKeys(), timeSeries, - monitoredResourceBuilder.build())); + clientId, + monitoredResource)); } } @@ -111,8 +76,6 @@ public void export(Collection metrics) { .build(); this.metricServiceClient.createServiceTimeSeries(request); - } catch (ApiException e) { - logger.log(Level.WARNING, "ApiException thrown when exporting TimeSeries.", e); } catch (Throwable e) { logger.log(Level.WARNING, "Exception thrown when exporting TimeSeries.", e); } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 25026527bb4f..a64234b81709 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -20,6 +20,7 @@ import com.google.api.MetricDescriptor.MetricKind; import com.google.api.MonitoredResource; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.monitoring.v3.TimeInterval; import com.google.monitoring.v3.TimeSeries; @@ -39,8 +40,10 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.security.SecureRandom; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -59,7 +62,15 @@ class BigtableStackdriverExportUtils { private static final String DOMAIN = "bigtable.googleapis.com/internal/client/"; - static String generateDefaultTaskValue() { + private static final Set PROMOTED_RESOURCE_LABELS = + ImmutableSet.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), + BuiltinMeasureConstants.INSTANCE_ID.getName(), + BuiltinMeasureConstants.CLUSTER.getName(), + BuiltinMeasureConstants.ZONE.getName(), + BuiltinMeasureConstants.TABLE.getName()); + + static String getDefaultTaskValue() { String jvmName = ManagementFactory.getRuntimeMXBean().getName(); if (jvmName.indexOf(64) < 1) { String hostname = "localhost"; @@ -78,29 +89,33 @@ static String generateDefaultTaskValue() { @VisibleForTesting static MetricKind createMetricKind(Type type) { - if (type != Type.GAUGE_INT64 && type != Type.GAUGE_DOUBLE) { - return type != Type.CUMULATIVE_INT64 - && type != Type.CUMULATIVE_DOUBLE - && type != Type.CUMULATIVE_DISTRIBUTION - ? MetricKind.UNRECOGNIZED - : MetricKind.CUMULATIVE; - } else { - return MetricKind.GAUGE; + switch (type) { + case GAUGE_INT64: + case GAUGE_DOUBLE: + return MetricKind.GAUGE; + case CUMULATIVE_DOUBLE: + case CUMULATIVE_INT64: + case CUMULATIVE_DISTRIBUTION: + return MetricKind.CUMULATIVE; + default: + return MetricKind.UNRECOGNIZED; } } @VisibleForTesting static com.google.api.MetricDescriptor.ValueType createValueType(Type type) { - if (type != Type.CUMULATIVE_DOUBLE && type != Type.GAUGE_DOUBLE) { - if (type != Type.GAUGE_INT64 && type != Type.CUMULATIVE_INT64) { - return type != Type.GAUGE_DISTRIBUTION && type != Type.CUMULATIVE_DISTRIBUTION - ? com.google.api.MetricDescriptor.ValueType.UNRECOGNIZED - : com.google.api.MetricDescriptor.ValueType.DISTRIBUTION; - } else { + switch (type) { + case GAUGE_DOUBLE: + case CUMULATIVE_DOUBLE: + return com.google.api.MetricDescriptor.ValueType.DOUBLE; + case GAUGE_INT64: + case CUMULATIVE_INT64: return com.google.api.MetricDescriptor.ValueType.INT64; - } - } else { - return com.google.api.MetricDescriptor.ValueType.DOUBLE; + case GAUGE_DISTRIBUTION: + case CUMULATIVE_DISTRIBUTION: + return com.google.api.MetricDescriptor.ValueType.DISTRIBUTION; + default: + return com.google.api.MetricDescriptor.ValueType.UNRECOGNIZED; } } @@ -108,15 +123,33 @@ static TimeSeries convertTimeSeries( String metricName, io.opencensus.metrics.export.MetricDescriptor.Type metricType, List labelKeys, - List labelValues, io.opencensus.metrics.export.TimeSeries timeSeries, + String clientId, MonitoredResource monitoredResource) { + MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder(); + + List metricTagKeys = new ArrayList<>(); + List metricTagValues = new ArrayList<>(); + + List labelValues = timeSeries.getLabelValues(); + for (int i = 0; i < labelValues.size(); i++) { + if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { + monitoredResourceBuilder.putLabels( + labelKeys.get(i).getKey(), labelValues.get(i).getValue()); + } else { + metricTagKeys.add(labelKeys.get(i)); + metricTagValues.add(labelValues.get(i)); + } + } + metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_ID.getName(), "client id")); + metricTagValues.add(LabelValue.create(clientId)); + TimeSeries.Builder builder = TimeSeries.newBuilder(); builder.setMetricKind(createMetricKind(metricType)); - builder.setResource(monitoredResource); + builder.setResource(monitoredResourceBuilder.build()); builder.setValueType(createValueType(metricType)); - builder.setMetric(createMetric(metricName, labelKeys, labelValues)); + builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); io.opencensus.common.Timestamp startTimeStamp = timeSeries.getStartTimestamp(); for (io.opencensus.metrics.export.Point point : timeSeries.getPoints()) { builder.addPoints(createPoint(point, startTimeStamp)); diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 2fd47c372b8c..60730a66ba0b 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -31,9 +31,7 @@ import java.io.IOException; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; -@ThreadSafe @InternalApi public class BigtableStackdriverStatsExporter { static final Object monitor = new Object(); @@ -42,7 +40,6 @@ public class BigtableStackdriverStatsExporter { @GuardedBy("monitor") private static BigtableStackdriverStatsExporter instance = null; - private static final String EXPORTER_SPAN_NAME = "BigtableExportMetricsToStackdriver"; private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); private static final String RESOURCE_TYPE = "bigtable_client_raw"; @@ -63,7 +60,6 @@ private BigtableStackdriverStatsExporter( io.opencensus.exporter.metrics.util.MetricReader.Options.builder() .setMetricProducerManager( Metrics.getExportComponent().getMetricProducerManager()) - .setSpanName(EXPORTER_SPAN_NAME) .build()), intervalMetricReaderOptionsBuilder.build()); } diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 25b5a0b97ea2..5ddcb69ec116 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -37,11 +37,14 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +@RunWith(JUnit4.class) public class BigtableCreateTimeSeriesExporterTest { private static final String projectId = "fake-project"; diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index d2d61565c683..c672d2f3472d 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -64,7 +64,7 @@ com.google.cloud google-cloud-bigtable-stats - + io.opencensus diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 81a15950904b..966fdda4e3a5 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -28,6 +28,7 @@ import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; import com.google.cloud.bigtable.stats.BigtableStackdriverStatsExporter; +import com.google.cloud.bigtable.stats.BuiltinViews; import com.google.common.base.MoreObjects; import com.google.common.base.Strings; import io.grpc.ManagedChannelBuilder; @@ -481,6 +482,7 @@ public Long getTargetRpcLatencyMsForBatchMutation() { @BetaApi("Built in metric is not currently stable and may change in the future") public Builder registerBuiltinMetrics() throws IOException { if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { + BuiltinViews.registerBigtableBuiltinViews(); BigtableStackdriverStatsExporter.register( stubSettings.getCredentialsProvider().getCredentials(), stubSettings.getProjectId()); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index fed6d3cc00a8..c6591d588a49 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -92,7 +92,6 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; -import com.google.cloud.bigtable.stats.BuiltinViews; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -232,7 +231,6 @@ public static EnhancedBigtableStubSettings finalizeSettings( BuiltinMetricsTracerFactory.create(builtinAttributes), // Add user configured tracer settings.getTracerFactory()))); - BuiltinViews.registerBigtableBuiltinViews(); return builder.build(); } From f5678b4497ffc403b329278d6195ffc9db2f4039 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 15 Jul 2022 12:05:26 -0400 Subject: [PATCH 11/30] remove unused setting --- .../cloud/bigtable/stats/BigtableStackdriverStatsExporter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 60730a66ba0b..6cb24505b56d 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -90,7 +90,6 @@ static MetricServiceClient createMetricServiceClient( org.threeten.bp.Duration stackdriverDuration = org.threeten.bp.Duration.ofMillis(deadline.toMillis()); - settingsBuilder.createMetricDescriptorSettings().setSimpleTimeoutNoRetries(stackdriverDuration); settingsBuilder.createTimeSeriesSettings().setSimpleTimeoutNoRetries(stackdriverDuration); return MetricServiceClient.create(settingsBuilder.build()); } From 1d77308cfa8213c471eb9f92ac3ba60f15d8b434 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 26 Jul 2022 19:13:04 -0400 Subject: [PATCH 12/30] make metrics consistent with cloud monitoring --- .../stats/BigtableStackdriverExportUtils.java | 6 ++---- .../stats/BigtableStackdriverStatsExporter.java | 2 +- .../bigtable/stats/BuiltinMeasureConstants.java | 5 ++--- .../bigtable/stats/BuiltinViewConstants.java | 16 ++++------------ .../BigtableCreateTimeSeriesExporterTest.java | 2 +- .../data/v2/stub/EnhancedBigtableStub.java | 12 ++++++------ .../metrics/BigtableTracerStreamingCallable.java | 1 + .../metrics/BigtableTracerUnaryCallable.java | 1 + 8 files changed, 18 insertions(+), 27 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index a64234b81709..20fe21c5c205 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -60,8 +60,6 @@ class BigtableStackdriverExportUtils { private static final io.opencensus.common.Function bucketOptionsExplicitFunction; - private static final String DOMAIN = "bigtable.googleapis.com/internal/client/"; - private static final Set PROMOTED_RESOURCE_LABELS = ImmutableSet.of( BuiltinMeasureConstants.PROJECT_ID.getName(), @@ -142,7 +140,7 @@ static TimeSeries convertTimeSeries( metricTagValues.add(labelValues.get(i)); } } - metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_ID.getName(), "client id")); + metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id")); metricTagValues.add(LabelValue.create(clientId)); TimeSeries.Builder builder = TimeSeries.newBuilder(); @@ -161,7 +159,7 @@ static TimeSeries convertTimeSeries( static com.google.api.Metric createMetric( String metricName, List labelKeys, List labelValues) { com.google.api.Metric.Builder builder = com.google.api.Metric.newBuilder(); - builder.setType(DOMAIN + metricName); + builder.setType(metricName); Map stringTagMap = Maps.newHashMap(); for (int i = 0; i < labelValues.size(); ++i) { diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 6cb24505b56d..199cf7936e4d 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -40,7 +40,7 @@ public class BigtableStackdriverStatsExporter { @GuardedBy("monitor") private static BigtableStackdriverStatsExporter instance = null; - private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); + private static final Duration EXPORT_INTERVAL = Duration.create(60, 0); private static final String RESOURCE_TYPE = "bigtable_client_raw"; private final IntervalMetricReader intervalMetricReader; diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java index 06ca674ffc04..2f51204d4b45 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java @@ -23,12 +23,11 @@ class BuiltinMeasureConstants { // Monitored resource TagKeys static final TagKey PROJECT_ID = TagKey.create("project_id"); - static final TagKey INSTANCE_ID = TagKey.create("instance_id"); + static final TagKey INSTANCE_ID = TagKey.create("instance"); static final TagKey CLUSTER = TagKey.create("cluster"); static final TagKey TABLE = TagKey.create("table"); static final TagKey ZONE = TagKey.create("zone"); - // Placeholder TagKey to be used in Stackdriver exporter - static final TagKey CLIENT_ID = TagKey.create("client_id"); + static final TagKey CLIENT_UID = TagKey.create("client_uid"); // Metrics TagKeys static final TagKey APP_PROFILE = TagKey.create("app_profile"); diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java index beceeeab831a..7c9dc34d788d 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinViewConstants.java @@ -59,7 +59,7 @@ class BuiltinViewConstants { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 100.0))); - private static final Aggregation AGGREGATION_ERROR_COUNT = Sum.create(); + private static final Aggregation AGGREGATION_COUNT = Sum.create(); static final View OPERATION_LATENCIES_VIEW = View.create( @@ -102,7 +102,7 @@ class BuiltinViewConstants { View.Name.create("bigtable.googleapis.com/internal/client/retry_count"), "The number of additional RPCs sent after the initial attempt.", RETRY_COUNT, - AGGREGATION_RETRY_COUNT, + AGGREGATION_COUNT, ImmutableList.of( PROJECT_ID, INSTANCE_ID, @@ -154,7 +154,7 @@ class BuiltinViewConstants { View.Name.create("bigtable.googleapis.com/internal/client/connectivity_error_count"), "Number of requests that failed to reach the Google datacenter. (Requests without google response headers).", CONNECTIVITY_ERROR_COUNT, - AGGREGATION_ERROR_COUNT, + AGGREGATION_COUNT, ImmutableList.of( PROJECT_ID, INSTANCE_ID, @@ -173,15 +173,7 @@ class BuiltinViewConstants { APPLICATION_LATENCIES, AGGREGATION_WITH_MILLIS_HISTOGRAM, ImmutableList.of( - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE, - METHOD, - STREAMING, - CLIENT_NAME, - CLUSTER, - ZONE, - TABLE)); + PROJECT_ID, INSTANCE_ID, APP_PROFILE, METHOD, CLIENT_NAME, CLUSTER, ZONE, TABLE)); static final View THROTTLING_LATENCIES_VIEW = View.create( diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 5ddcb69ec116..6f97959e09f9 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -134,7 +134,7 @@ public void testTimeSeries() { assertThat(timeSeries.getMetric().getLabelsMap()) .containsAtLeast(BuiltinMeasureConstants.APP_PROFILE.getName(), appProfileId); assertThat(timeSeries.getMetric().getLabelsMap()) - .containsKey(BuiltinMeasureConstants.CLIENT_ID.getName()); + .containsKey(BuiltinMeasureConstants.CLIENT_UID.getName()); assertThat(timeSeries.getPoints(0).getValue().getDoubleValue()).isEqualTo(fakeValue); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index c6591d588a49..880522d74db8 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -201,7 +201,7 @@ public static EnhancedBigtableStubSettings finalizeSettings( ImmutableMap builtinAttributes = ImmutableMap.builder() .put("project_id", settings.getProjectId()) - .put("instance_id", settings.getInstanceId()) + .put("instance", settings.getInstanceId()) .put("app_profile", settings.getAppProfileId()) .build(); // Inject Opencensus instrumentation @@ -335,10 +335,13 @@ public ServerStreamingCallable createReadRowsCallable( ServerStreamingCallable readRowsUserCallable = new ReadRowsUserCallable<>(readRowsCallable, requestContext); + ServerStreamingCallable withBigtableTracer = + new BigtableTracerStreamingCallable<>(readRowsUserCallable); + SpanName span = getSpanName("ReadRows"); ServerStreamingCallable traced = new TracedServerStreamingCallable<>( - readRowsUserCallable, clientContext.getTracerFactory(), span); + withBigtableTracer, clientContext.getTracerFactory(), span); return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } @@ -438,13 +441,10 @@ public Map extract(ReadRowsRequest readRowsRequest) { ServerStreamingCallable watched = Callables.watched(merging, innerSettings, clientContext); - ServerStreamingCallable withBigtableTracer = - new BigtableTracerStreamingCallable<>(watched); - // Retry logic is split into 2 parts to workaround a rare edge case described in // ReadRowsRetryCompletedCallable ServerStreamingCallable retrying1 = - new ReadRowsRetryCompletedCallable<>(withBigtableTracer); + new ReadRowsRetryCompletedCallable<>(watched); ServerStreamingCallable retrying2 = Callables.retrying(retrying1, innerSettings, clientContext); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index c7f09c4db18e..3de427d2e451 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -22,6 +22,7 @@ import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.StreamController; import com.google.bigtable.v2.ResponseParams; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index 50d24ecbaf6b..9e7034b440cc 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -23,6 +23,7 @@ import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.UnaryCallable; import com.google.bigtable.v2.ResponseParams; +import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.InvalidProtocolBufferException; From 6c92a39ce942a8495a10ec8c095026a06cd2b5a7 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 1 Aug 2022 18:06:44 -0400 Subject: [PATCH 13/30] convert undefined to global --- .../stats/BigtableStackdriverExportUtils.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 20fe21c5c205..90f56d38903b 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -133,13 +133,22 @@ static TimeSeries convertTimeSeries( List labelValues = timeSeries.getLabelValues(); for (int i = 0; i < labelValues.size(); i++) { if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { - monitoredResourceBuilder.putLabels( - labelKeys.get(i).getKey(), labelValues.get(i).getValue()); + if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.CLUSTER.getName()) && labelValues.get(i).getValue().equals("undefined")) { + monitoredResourceBuilder.putLabels( + labelKeys.get(i).getKey(), "global"); + } else if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.ZONE.getName()) && labelValues.get(i).getValue().equals("undefined")) { + monitoredResourceBuilder.putLabels( + labelKeys.get(i).getKey(), "global"); + } else { + monitoredResourceBuilder.putLabels( + labelKeys.get(i).getKey(), labelValues.get(i).getValue()); + } } else { metricTagKeys.add(labelKeys.get(i)); metricTagValues.add(labelValues.get(i)); } } + labelKeys.add(LabelKey.create("uid")); metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id")); metricTagValues.add(LabelValue.create(clientId)); From c3afccb7cd17434e23159c38acc5768f892e76dd Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 2 Aug 2022 09:56:07 -0400 Subject: [PATCH 14/30] update --- .../stats/BigtableStackdriverExportUtils.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 90f56d38903b..1d36adbf700f 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -133,22 +133,21 @@ static TimeSeries convertTimeSeries( List labelValues = timeSeries.getLabelValues(); for (int i = 0; i < labelValues.size(); i++) { if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { - if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.CLUSTER.getName()) && labelValues.get(i).getValue().equals("undefined")) { - monitoredResourceBuilder.putLabels( - labelKeys.get(i).getKey(), "global"); - } else if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.ZONE.getName()) && labelValues.get(i).getValue().equals("undefined")) { - monitoredResourceBuilder.putLabels( - labelKeys.get(i).getKey(), "global"); + if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.CLUSTER.getName()) + && labelValues.get(i).getValue().equals("undefined")) { + monitoredResourceBuilder.putLabels(labelKeys.get(i).getKey(), "global"); + } else if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.ZONE.getName()) + && labelValues.get(i).getValue().equals("undefined")) { + monitoredResourceBuilder.putLabels(labelKeys.get(i).getKey(), "global"); } else { monitoredResourceBuilder.putLabels( - labelKeys.get(i).getKey(), labelValues.get(i).getValue()); + labelKeys.get(i).getKey(), labelValues.get(i).getValue()); } } else { metricTagKeys.add(labelKeys.get(i)); metricTagValues.add(labelValues.get(i)); } } - labelKeys.add(LabelKey.create("uid")); metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id")); metricTagValues.add(LabelValue.create(clientId)); From c9666716c774b0e93aca76eb9d7089473a2649f1 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 2 Aug 2022 11:31:55 -0400 Subject: [PATCH 15/30] add bigtable tracer back in the base callable --- .../bigtable/data/v2/stub/EnhancedBigtableStub.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 880522d74db8..574890fc0972 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -335,13 +335,10 @@ public ServerStreamingCallable createReadRowsCallable( ServerStreamingCallable readRowsUserCallable = new ReadRowsUserCallable<>(readRowsCallable, requestContext); - ServerStreamingCallable withBigtableTracer = - new BigtableTracerStreamingCallable<>(readRowsUserCallable); - SpanName span = getSpanName("ReadRows"); ServerStreamingCallable traced = new TracedServerStreamingCallable<>( - withBigtableTracer, clientContext.getTracerFactory(), span); + readRowsUserCallable, clientContext.getTracerFactory(), span); return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } @@ -441,10 +438,13 @@ public Map extract(ReadRowsRequest readRowsRequest) { ServerStreamingCallable watched = Callables.watched(merging, innerSettings, clientContext); + ServerStreamingCallable withBigtableTracer = + new BigtableTracerStreamingCallable<>(watched); + // Retry logic is split into 2 parts to workaround a rare edge case described in // ReadRowsRetryCompletedCallable ServerStreamingCallable retrying1 = - new ReadRowsRetryCompletedCallable<>(watched); + new ReadRowsRetryCompletedCallable<>(withBigtableTracer); ServerStreamingCallable retrying2 = Callables.retrying(retrying1, innerSettings, clientContext); From aee2e15d9c6bcfd9518a923507617d150a59b18a Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 2 Aug 2022 11:33:04 -0400 Subject: [PATCH 16/30] fix format --- .../cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 574890fc0972..3de22888be0c 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -439,7 +439,7 @@ public Map extract(ReadRowsRequest readRowsRequest) { Callables.watched(merging, innerSettings, clientContext); ServerStreamingCallable withBigtableTracer = - new BigtableTracerStreamingCallable<>(watched); + new BigtableTracerStreamingCallable<>(watched); // Retry logic is split into 2 parts to workaround a rare edge case described in // ReadRowsRetryCompletedCallable From 8129f0218e71049aa0bcf4b32953ec9486cc5f6c Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 2 Aug 2022 11:45:29 -0400 Subject: [PATCH 17/30] fix the tag name --- .../google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java index 9b486f919f2d..929ee85f481a 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/ITBuiltinViewConstantsTest.java @@ -32,7 +32,7 @@ public void testBasicTagsExistForAllViews() { assertWithMessage(view + " should have all basic tags") .that(viewToTagMap.get(view)) .containsAtLeast( - "project_id", "instance_id", "app_profile", "method", "zone", "cluster", "table"); + "project_id", "instance", "app_profile", "method", "zone", "cluster", "table"); } } } From 551cff87dc99db96a4dece95434a5f7527df9f42 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 3 Aug 2022 15:26:09 -0400 Subject: [PATCH 18/30] add the link to the form --- .../google/cloud/bigtable/data/v2/BigtableDataSettings.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 966fdda4e3a5..80837e8a1582 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -478,7 +478,11 @@ public Long getTargetRpcLatencyMsForBatchMutation() { return stubSettings.bulkMutateRowsSettings().getTargetRpcLatencyMs(); } - /** Register built in metrics. * */ + /** + * Register built in metrics. This is an experimental feature. Please fill up this form + * to have your project allow listed for the private preview: + * https://forms.gle/xuhu6vCunn2MjV2m9 + */ @BetaApi("Built in metric is not currently stable and may change in the future") public Builder registerBuiltinMetrics() throws IOException { if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { From da85a107cc34cfa93c73a7564330c9d97baed32a Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 3 Aug 2022 15:28:35 -0400 Subject: [PATCH 19/30] fix format --- .../google/cloud/bigtable/data/v2/BigtableDataSettings.java | 5 ++--- .../v2/stub/metrics/BigtableTracerStreamingCallable.java | 1 - .../data/v2/stub/metrics/BigtableTracerUnaryCallable.java | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 80837e8a1582..57ee2f124fd1 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -479,9 +479,8 @@ public Long getTargetRpcLatencyMsForBatchMutation() { } /** - * Register built in metrics. This is an experimental feature. Please fill up this form - * to have your project allow listed for the private preview: - * https://forms.gle/xuhu6vCunn2MjV2m9 + * Register built in metrics. This is an experimental feature. Please fill up this form to have + * your project allow listed for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 */ @BetaApi("Built in metric is not currently stable and may change in the future") public Builder registerBuiltinMetrics() throws IOException { diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index 3de427d2e451..c7f09c4db18e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -22,7 +22,6 @@ import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.StreamController; import com.google.bigtable.v2.ResponseParams; -import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index 9e7034b440cc..50d24ecbaf6b 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -23,7 +23,6 @@ import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.UnaryCallable; import com.google.bigtable.v2.ResponseParams; -import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.InvalidProtocolBufferException; From 541feb1c65898c2d177e8876e004be63bc657981 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 4 Aug 2022 13:50:37 -0400 Subject: [PATCH 20/30] fix dependency conflicts --- google-cloud-bigtable-stats/pom.xml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index b32a9fcbd1db..a8edb3fbfcbd 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -18,7 +18,7 @@ 0.31.1 - 3.3.1 + 3.3.6 @@ -68,6 +68,17 @@ com.google.cloud google-cloud-monitoring + + + + com.google.http-client + google-http-client + + + com.google.http-client + google-http-client-gson + + com.google.api.grpc From d8f6162e3c23f10270ce75cdf009a7eadd335dca Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 4 Aug 2022 15:29:40 -0400 Subject: [PATCH 21/30] fix image tests --- google-cloud-bigtable-stats/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index a8edb3fbfcbd..4ea560fde783 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -132,6 +132,12 @@ jsr305 + + com.google.http-client + google-http-client + runtime + + com.google.truth From b2192006f859441e58d7f8f98a4da67bd87a05c7 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 5 Aug 2022 11:25:41 -0400 Subject: [PATCH 22/30] update undefined cluster to global --- google-cloud-bigtable-stats/pom.xml | 6 +++--- .../stats/BigtableStackdriverExportUtils.java | 12 ++--------- .../BigtableStackdriverStatsExporter.java | 2 +- google-cloud-bigtable/pom.xml | 20 +++++++++---------- .../v2/stub/metrics/BuiltinMetricsTracer.java | 4 ++-- .../metrics/BuiltinMetricsTracerTest.java | 4 ++-- 6 files changed, 20 insertions(+), 28 deletions(-) diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index 4ea560fde783..10b64dc4667f 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -68,15 +68,14 @@ com.google.cloud google-cloud-monitoring - com.google.http-client - google-http-client + google-http-client-gson com.google.http-client - google-http-client-gson + google-http-client @@ -132,6 +131,7 @@ jsr305 + com.google.http-client google-http-client diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 1d36adbf700f..20fe21c5c205 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -133,16 +133,8 @@ static TimeSeries convertTimeSeries( List labelValues = timeSeries.getLabelValues(); for (int i = 0; i < labelValues.size(); i++) { if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { - if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.CLUSTER.getName()) - && labelValues.get(i).getValue().equals("undefined")) { - monitoredResourceBuilder.putLabels(labelKeys.get(i).getKey(), "global"); - } else if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.ZONE.getName()) - && labelValues.get(i).getValue().equals("undefined")) { - monitoredResourceBuilder.putLabels(labelKeys.get(i).getKey(), "global"); - } else { - monitoredResourceBuilder.putLabels( - labelKeys.get(i).getKey(), labelValues.get(i).getValue()); - } + monitoredResourceBuilder.putLabels( + labelKeys.get(i).getKey(), labelValues.get(i).getValue()); } else { metricTagKeys.add(labelKeys.get(i)); metricTagValues.add(labelValues.get(i)); diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 199cf7936e4d..6cb24505b56d 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -40,7 +40,7 @@ public class BigtableStackdriverStatsExporter { @GuardedBy("monitor") private static BigtableStackdriverStatsExporter instance = null; - private static final Duration EXPORT_INTERVAL = Duration.create(60, 0); + private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); private static final String RESOURCE_TYPE = "bigtable_client_raw"; private final IntervalMetricReader intervalMetricReader; diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index c672d2f3472d..4872466846ba 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -146,6 +146,16 @@ grpc-alts runtime + + com.google.http-client + google-http-client + runtime + + + com.google.http-client + google-http-client-gson + runtime + com.google.http-client @@ -192,6 +184,17 @@ org.apache.maven.plugins maven-dependency-plugin 3.3.0 + + + + + + + + io.opencensus:opencensus-exporter-metrics-util:* + io.opencensus:opencensus-exporter-stats-stackdriver:* + + org.codehaus.mojo @@ -218,7 +221,7 @@ - io.opencensus:*:[${opencensus.version}] + io.opencensus:*:[0.31.1] io.opencensus:opencensus-proto:[0.2.0] diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index 8f118c02eb20..67550fb5120a 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -19,9 +19,9 @@ import com.google.cloud.monitoring.v3.MetricServiceClient; import com.google.monitoring.v3.CreateTimeSeriesRequest; import com.google.monitoring.v3.ProjectName; -import com.google.monitoring.v3.TimeSeries; import io.opencensus.exporter.metrics.util.MetricExporter; import io.opencensus.metrics.export.Metric; +import io.opencensus.metrics.export.TimeSeries; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -48,7 +48,7 @@ final class BigtableCreateTimeSeriesExporter extends MetricExporter { } public void export(Collection metrics) { - List timeSeriesList = new ArrayList(metrics.size()); + List timeSeriesList = new ArrayList(metrics.size()); for (Metric metric : metrics) { // only export bigtable metrics @@ -56,15 +56,10 @@ public void export(Collection metrics) { continue; } - for (io.opencensus.metrics.export.TimeSeries timeSeries : metric.getTimeSeriesList()) { + for (TimeSeries timeSeries : metric.getTimeSeriesList()) { timeSeriesList.add( BigtableStackdriverExportUtils.convertTimeSeries( - metric.getMetricDescriptor().getName(), - metric.getMetricDescriptor().getType(), - metric.getMetricDescriptor().getLabelKeys(), - timeSeries, - clientId, - monitoredResource)); + metric.getMetricDescriptor(), timeSeries, clientId, monitoredResource)); } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 20fe21c5c205..3dfbcaaf252e 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -17,24 +17,26 @@ import com.google.api.Distribution.BucketOptions; import com.google.api.Distribution.BucketOptions.Explicit; +import com.google.api.Metric; import com.google.api.MetricDescriptor.MetricKind; import com.google.api.MonitoredResource; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.monitoring.v3.TimeInterval; -import com.google.monitoring.v3.TimeSeries; import com.google.monitoring.v3.TypedValue; -import com.google.protobuf.Any; -import com.google.protobuf.ByteString; +import io.opencensus.common.Function; import io.opencensus.common.Functions; +import io.opencensus.common.Timestamp; import io.opencensus.metrics.LabelKey; import io.opencensus.metrics.LabelValue; -import io.opencensus.metrics.data.AttachmentValue; +import io.opencensus.metrics.export.Distribution; import io.opencensus.metrics.export.Distribution.Bucket; import io.opencensus.metrics.export.Distribution.BucketOptions.ExplicitOptions; +import io.opencensus.metrics.export.MetricDescriptor; import io.opencensus.metrics.export.MetricDescriptor.Type; +import io.opencensus.metrics.export.Point; import io.opencensus.metrics.export.Summary; +import io.opencensus.metrics.export.TimeSeries; import io.opencensus.metrics.export.Value; import java.lang.management.ManagementFactory; import java.net.InetAddress; @@ -49,17 +51,45 @@ import javax.annotation.Nullable; class BigtableStackdriverExportUtils { - private static final Logger logger; - private static final io.opencensus.common.Function typedValueDoubleFunction; - private static final io.opencensus.common.Function typedValueLongFunction; - private static final io.opencensus.common.Function< - io.opencensus.metrics.export.Distribution, TypedValue> - typedValueDistributionFunction; - private static final io.opencensus.common.Function typedValueSummaryFunction; - private static final io.opencensus.common.Function - bucketOptionsExplicitFunction; + private static final Logger logger = + Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); + private static final Function typedValueDoubleFunction = + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + builder.setDoubleValue(arg); + return builder.build(); + }; + private static final Function typedValueLongFunction = + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + builder.setInt64Value(arg); + return builder.build(); + }; + private static final Function typedValueDistributionFunction = + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + return builder + .setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg)) + .build(); + }; + private static final Function typedValueSummaryFunction = + arg -> { + TypedValue.Builder builder = TypedValue.newBuilder(); + return builder.build(); + }; + private static final Function bucketOptionsExplicitFunction = + arg -> { + BucketOptions.Builder builder = BucketOptions.newBuilder(); + Explicit.Builder explicitBuilder = Explicit.newBuilder(); + explicitBuilder.addBounds(0.0D); + explicitBuilder.addAllBounds(arg.getBucketBoundaries()); + builder.setExplicitBuckets(explicitBuilder.build()); + return builder.build(); + }; + + // promote the following metric labels to monitored resource labels private static final Set PROMOTED_RESOURCE_LABELS = ImmutableSet.of( BuiltinMeasureConstants.PROJECT_ID.getName(), @@ -68,6 +98,51 @@ class BigtableStackdriverExportUtils { BuiltinMeasureConstants.ZONE.getName(), BuiltinMeasureConstants.TABLE.getName()); + private static final LabelKey clientIdLabelKey = + LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id"); + + static com.google.monitoring.v3.TimeSeries convertTimeSeries( + MetricDescriptor metricDescriptor, + TimeSeries timeSeries, + String clientId, + MonitoredResource monitoredResource) { + String metricName = metricDescriptor.getName(); + List labelKeys = metricDescriptor.getLabelKeys(); + Type metricType = metricDescriptor.getType(); + + MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder(); + + List metricTagKeys = new ArrayList<>(); + List metricTagValues = new ArrayList<>(); + + List labelValues = timeSeries.getLabelValues(); + for (int i = 0; i < labelValues.size(); i++) { + // If the label is defined in the monitored resource, convert it to + // a monitored resource label. Otherwise, keep it as a metric label. + if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { + monitoredResourceBuilder.putLabels( + labelKeys.get(i).getKey(), labelValues.get(i).getValue()); + } else { + metricTagKeys.add(labelKeys.get(i)); + metricTagValues.add(labelValues.get(i)); + } + } + metricTagKeys.add(clientIdLabelKey); + metricTagValues.add(LabelValue.create(clientId)); + + com.google.monitoring.v3.TimeSeries.Builder builder = + com.google.monitoring.v3.TimeSeries.newBuilder(); + builder.setResource(monitoredResourceBuilder.build()); + builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); + builder.setMetricKind(createMetricKind(metricType)); + builder.setValueType(createValueType(metricType)); + Timestamp startTimeStamp = timeSeries.getStartTimestamp(); + for (io.opencensus.metrics.export.Point point : timeSeries.getPoints()) { + builder.addPoints(createPoint(point, startTimeStamp)); + } + return builder.build(); + } + static String getDefaultTaskValue() { String jvmName = ManagementFactory.getRuntimeMXBean().getName(); if (jvmName.indexOf(64) < 1) { @@ -85,8 +160,7 @@ static String getDefaultTaskValue() { } } - @VisibleForTesting - static MetricKind createMetricKind(Type type) { + private static MetricKind createMetricKind(Type type) { switch (type) { case GAUGE_INT64: case GAUGE_DOUBLE: @@ -100,8 +174,7 @@ static MetricKind createMetricKind(Type type) { } } - @VisibleForTesting - static com.google.api.MetricDescriptor.ValueType createValueType(Type type) { + private static com.google.api.MetricDescriptor.ValueType createValueType(Type type) { switch (type) { case GAUGE_DOUBLE: case CUMULATIVE_DOUBLE: @@ -117,48 +190,9 @@ static com.google.api.MetricDescriptor.ValueType createValueType(Type type) { } } - static TimeSeries convertTimeSeries( - String metricName, - io.opencensus.metrics.export.MetricDescriptor.Type metricType, - List labelKeys, - io.opencensus.metrics.export.TimeSeries timeSeries, - String clientId, - MonitoredResource monitoredResource) { - - MonitoredResource.Builder monitoredResourceBuilder = monitoredResource.toBuilder(); - - List metricTagKeys = new ArrayList<>(); - List metricTagValues = new ArrayList<>(); - - List labelValues = timeSeries.getLabelValues(); - for (int i = 0; i < labelValues.size(); i++) { - if (PROMOTED_RESOURCE_LABELS.contains(labelKeys.get(i).getKey())) { - monitoredResourceBuilder.putLabels( - labelKeys.get(i).getKey(), labelValues.get(i).getValue()); - } else { - metricTagKeys.add(labelKeys.get(i)); - metricTagValues.add(labelValues.get(i)); - } - } - metricTagKeys.add(LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id")); - metricTagValues.add(LabelValue.create(clientId)); - - TimeSeries.Builder builder = TimeSeries.newBuilder(); - builder.setMetricKind(createMetricKind(metricType)); - builder.setResource(monitoredResourceBuilder.build()); - builder.setValueType(createValueType(metricType)); - builder.setMetric(createMetric(metricName, metricTagKeys, metricTagValues)); - io.opencensus.common.Timestamp startTimeStamp = timeSeries.getStartTimestamp(); - for (io.opencensus.metrics.export.Point point : timeSeries.getPoints()) { - builder.addPoints(createPoint(point, startTimeStamp)); - } - return builder.build(); - } - - @VisibleForTesting - static com.google.api.Metric createMetric( + private static Metric createMetric( String metricName, List labelKeys, List labelValues) { - com.google.api.Metric.Builder builder = com.google.api.Metric.newBuilder(); + Metric.Builder builder = Metric.newBuilder(); builder.setType(metricName); Map stringTagMap = Maps.newHashMap(); @@ -173,15 +207,10 @@ static com.google.api.Metric createMetric( return builder.build(); } - @VisibleForTesting - static com.google.monitoring.v3.Point createPoint( - io.opencensus.metrics.export.Point point, - @Nullable io.opencensus.common.Timestamp startTimestamp) { + private static com.google.monitoring.v3.Point createPoint(Point point, Timestamp startTimestamp) { com.google.monitoring.v3.TimeInterval.Builder timeIntervalBuilder = TimeInterval.newBuilder(); + timeIntervalBuilder.setStartTime(convertTimestamp(startTimestamp)); timeIntervalBuilder.setEndTime(convertTimestamp(point.getTimestamp())); - if (startTimestamp != null) { - timeIntervalBuilder.setStartTime(convertTimestamp(startTimestamp)); - } com.google.monitoring.v3.Point.Builder builder = com.google.monitoring.v3.Point.newBuilder(); builder.setInterval(timeIntervalBuilder.build()); @@ -189,8 +218,7 @@ static com.google.monitoring.v3.Point createPoint( return builder.build(); } - @VisibleForTesting - static TypedValue createTypedValue(Value value) { + private static TypedValue createTypedValue(Value value) { return value.match( typedValueDoubleFunction, typedValueLongFunction, @@ -199,9 +227,7 @@ static TypedValue createTypedValue(Value value) { Functions.throwIllegalArgumentException()); } - @VisibleForTesting - static com.google.api.Distribution createDistribution( - io.opencensus.metrics.export.Distribution distribution) { + private static com.google.api.Distribution createDistribution(Distribution distribution) { com.google.api.Distribution.Builder builder = com.google.api.Distribution.newBuilder() .setBucketOptions(createBucketOptions(distribution.getBucketOptions())) @@ -211,13 +237,12 @@ static com.google.api.Distribution createDistribution( ? 0.0D : distribution.getSum() / (double) distribution.getCount()) .setSumOfSquaredDeviation(distribution.getSumOfSquaredDeviations()); - setBucketCountsAndExemplars(distribution.getBuckets(), builder); + setBucketCounts(distribution.getBuckets(), builder); return builder.build(); } - @VisibleForTesting - static BucketOptions createBucketOptions( - @Nullable io.opencensus.metrics.export.Distribution.BucketOptions bucketOptions) { + private static BucketOptions createBucketOptions( + @Nullable Distribution.BucketOptions bucketOptions) { com.google.api.Distribution.BucketOptions.Builder builder = BucketOptions.newBuilder(); return bucketOptions == null ? builder.build() @@ -225,44 +250,16 @@ static BucketOptions createBucketOptions( bucketOptionsExplicitFunction, Functions.throwIllegalArgumentException()); } - private static void setBucketCountsAndExemplars( + private static void setBucketCounts( List buckets, com.google.api.Distribution.Builder builder) { builder.addBucketCounts(0L); for (Bucket bucket : buckets) { builder.addBucketCounts(bucket.getCount()); - io.opencensus.metrics.data.Exemplar exemplar = bucket.getExemplar(); - if (exemplar != null) { - builder.addExemplars(toProtoExemplar(exemplar)); - } } } - private static com.google.api.Distribution.Exemplar toProtoExemplar( - io.opencensus.metrics.data.Exemplar exemplar) { - com.google.api.Distribution.Exemplar.Builder builder = - com.google.api.Distribution.Exemplar.newBuilder() - .setValue(exemplar.getValue()) - .setTimestamp(convertTimestamp(exemplar.getTimestamp())); - - for (Map.Entry attachment : exemplar.getAttachments().entrySet()) { - AttachmentValue value = attachment.getValue(); - builder.addAttachments(toProtoStringAttachment(value)); - } - - return builder.build(); - } - - private static Any toProtoStringAttachment(AttachmentValue attachmentValue) { - return Any.newBuilder() - .setTypeUrl("type.googleapis.com/google.protobuf.StringValue") - .setValue(ByteString.copyFromUtf8(attachmentValue.getValue())) - .build(); - } - - @VisibleForTesting - static com.google.protobuf.Timestamp convertTimestamp( - io.opencensus.common.Timestamp censusTimestamp) { + private static com.google.protobuf.Timestamp convertTimestamp(Timestamp censusTimestamp) { return censusTimestamp.getSeconds() < 0L ? com.google.protobuf.Timestamp.newBuilder().build() : com.google.protobuf.Timestamp.newBuilder() @@ -270,43 +267,4 @@ static com.google.protobuf.Timestamp convertTimestamp( .setNanos(censusTimestamp.getNanos()) .build(); } - - private BigtableStackdriverExportUtils() {} - - static { - logger = Logger.getLogger(BigtableStackdriverExportUtils.class.getName()); - typedValueDoubleFunction = - arg -> { - TypedValue.Builder builder = TypedValue.newBuilder(); - builder.setDoubleValue(arg); - return builder.build(); - }; - typedValueLongFunction = - arg -> { - TypedValue.Builder builder = TypedValue.newBuilder(); - builder.setInt64Value(arg); - return builder.build(); - }; - typedValueDistributionFunction = - arg -> { - TypedValue.Builder builder = TypedValue.newBuilder(); - return builder - .setDistributionValue(BigtableStackdriverExportUtils.createDistribution(arg)) - .build(); - }; - typedValueSummaryFunction = - arg -> { - TypedValue.Builder builder = TypedValue.newBuilder(); - return builder.build(); - }; - bucketOptionsExplicitFunction = - arg -> { - BucketOptions.Builder builder = BucketOptions.newBuilder(); - Explicit.Builder explicitBuilder = Explicit.newBuilder(); - explicitBuilder.addBounds(0.0D); - explicitBuilder.addAllBounds(arg.getBucketBoundaries()); - builder.setExplicitBuckets(explicitBuilder.build()); - return builder.build(); - }; - } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 6cb24505b56d..970a5bf61c4b 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -34,12 +34,13 @@ @InternalApi public class BigtableStackdriverStatsExporter { - static final Object monitor = new Object(); + static final Object lock = new Object(); @Nullable @GuardedBy("monitor") private static BigtableStackdriverStatsExporter instance = null; + // Default export interval is 10 minutes private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); private static final String RESOURCE_TYPE = "bigtable_client_raw"; @@ -57,7 +58,7 @@ private BigtableStackdriverStatsExporter( IntervalMetricReader.create( new BigtableCreateTimeSeriesExporter(projectId, metricServiceClient, monitoredResource), MetricReader.create( - io.opencensus.exporter.metrics.util.MetricReader.Options.builder() + MetricReader.Options.builder() .setMetricProducerManager( Metrics.getExportComponent().getMetricProducerManager()) .build()), @@ -66,9 +67,10 @@ private BigtableStackdriverStatsExporter( public static void register(@Nullable Credentials credentials, String projectId) throws IOException { - synchronized (monitor) { + synchronized (lock) { Preconditions.checkState( instance == null, "Bigtable Stackdriver stats exporter is already created"); + // Default timeout for creating a client is 1 minute MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); MonitoredResource resourceType = MonitoredResource.newBuilder().setType(RESOURCE_TYPE).build(); @@ -81,7 +83,7 @@ public static void register(@Nullable Credentials credentials, String projectId) @VisibleForTesting static MetricServiceClient createMetricServiceClient( @Nullable Credentials credentials, Duration deadline) throws IOException { - com.google.cloud.monitoring.v3.MetricServiceSettings.Builder settingsBuilder = + MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder() .setTransportChannelProvider(InstantiatingGrpcChannelProvider.newBuilder().build()); if (credentials != null) { @@ -95,7 +97,7 @@ static MetricServiceClient createMetricServiceClient( } public static void unregister() { - synchronized (monitor) { + synchronized (lock) { if (instance != null) { instance.intervalMetricReader.stop(); } diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 4872466846ba..a035d507e13f 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -146,6 +146,7 @@ grpc-alts runtime + com.google.http-client google-http-client diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 57ee2f124fd1..8520a10e3b3e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -23,6 +23,8 @@ import com.google.api.gax.core.NoCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.UnaryCallSettings; +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.stub.BigtableBatchingCallSettings; @@ -198,6 +200,42 @@ public static void enableGfeOpenCensusStats() { com.google.cloud.bigtable.data.v2.stub.metrics.RpcViews.registerBigtableClientGfeViews(); } + /** + * Register built in metrics. This is an experimental feature. Please fill up this form to have + * your project allow listed for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 + */ + @BetaApi("Built in metric is not currently stable and may change in the future") + public static void registerBuiltinMetrics(String projectId) throws IOException { + GoogleCredentials.getApplicationDefault(); + if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { + BuiltinViews.registerBigtableBuiltinViews(); + BigtableStackdriverStatsExporter.register( + GoogleCredentials.getApplicationDefault(), projectId); + } + } + + /** + * Register built in metrics with credentials. This is an experimental feature. Please fill up + * this form to have your project allow listed for the private preview: + * https://forms.gle/xuhu6vCunn2MjV2m9 + */ + @BetaApi("Built in metric is not currently stable and may change in the future") + public static void registerBuiltinMetrics(Credentials credentials, String projectId) + throws IOException { + if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { + BuiltinViews.registerBigtableBuiltinViews(); + BigtableStackdriverStatsExporter.register(credentials, projectId); + } + } + + /** Unregister built in metrics. * */ + @BetaApi("Built in metrics is not currently stable and may change in the future") + public static void unregisterBuiltinMetrics() { + if (BUILTIN_METRICS_REGISTERED.compareAndSet(true, false)) { + BigtableStackdriverStatsExporter.unregister(); + } + } + /** Returns the target project id. */ public String getProjectId() { return stubSettings.getProjectId(); @@ -253,7 +291,7 @@ public Long getBatchMutationsTargetRpcLatencyMs() { * @return */ @BetaApi("Built in metrics is not currently stable and may change in the future") - public boolean isBuiltinMetricsRegistered() { + public static boolean isBuiltinMetricsRegistered() { return BUILTIN_METRICS_REGISTERED.get(); } @@ -478,35 +516,6 @@ public Long getTargetRpcLatencyMsForBatchMutation() { return stubSettings.bulkMutateRowsSettings().getTargetRpcLatencyMs(); } - /** - * Register built in metrics. This is an experimental feature. Please fill up this form to have - * your project allow listed for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 - */ - @BetaApi("Built in metric is not currently stable and may change in the future") - public Builder registerBuiltinMetrics() throws IOException { - if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { - BuiltinViews.registerBigtableBuiltinViews(); - BigtableStackdriverStatsExporter.register( - stubSettings.getCredentialsProvider().getCredentials(), stubSettings.getProjectId()); - } - return this; - } - - /** Unregister built in metrics. * */ - @BetaApi("Built in metrics is not currently stable and may change in the future") - public Builder unregisterBuiltinMetrics() { - if (BUILTIN_METRICS_REGISTERED.compareAndSet(true, false)) { - BigtableStackdriverStatsExporter.unregister(); - } - return this; - } - - /** Gets if built in metrics are registered */ - @BetaApi("Built in metric is not currently stable and may change in the future") - public boolean isBuiltinMetricsRegistered() { - return BUILTIN_METRICS_REGISTERED.get(); - } - /** * Returns the underlying settings for making RPC calls. The settings should be changed with * care. From 8feaeecfe8dec42d99814adeee30e6b8487f2638 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 9 Aug 2022 15:16:44 -0400 Subject: [PATCH 24/30] tweak export interval --- .../BigtableStackdriverStatsExporter.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 970a5bf61c4b..d9d2ed94f159 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -37,11 +37,11 @@ public class BigtableStackdriverStatsExporter { static final Object lock = new Object(); @Nullable - @GuardedBy("monitor") + @GuardedBy("lock") private static BigtableStackdriverStatsExporter instance = null; - // Default export interval is 10 minutes - private static final Duration EXPORT_INTERVAL = Duration.create(600, 0); + // Default export interval is 5 minutes + private static final Duration EXPORT_INTERVAL = Duration.create(60 * 5, 0); private static final String RESOURCE_TYPE = "bigtable_client_raw"; private final IntervalMetricReader intervalMetricReader; @@ -65,8 +65,7 @@ private BigtableStackdriverStatsExporter( intervalMetricReaderOptionsBuilder.build()); } - public static void register(@Nullable Credentials credentials, String projectId) - throws IOException { + public static void register(Credentials credentials, String projectId) throws IOException { synchronized (lock) { Preconditions.checkState( instance == null, "Bigtable Stackdriver stats exporter is already created"); @@ -79,16 +78,14 @@ public static void register(@Nullable Credentials credentials, String projectId) } } - @GuardedBy("monitor") + @GuardedBy("lock") @VisibleForTesting - static MetricServiceClient createMetricServiceClient( - @Nullable Credentials credentials, Duration deadline) throws IOException { + static MetricServiceClient createMetricServiceClient(Credentials credentials, Duration deadline) + throws IOException { MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder() .setTransportChannelProvider(InstantiatingGrpcChannelProvider.newBuilder().build()); - if (credentials != null) { - settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); - } + settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); org.threeten.bp.Duration stackdriverDuration = org.threeten.bp.Duration.ofMillis(deadline.toMillis()); From e9608788f1358fb446d7572e7eae11db2eaa320f Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 10 Aug 2022 10:44:34 -0400 Subject: [PATCH 25/30] remove unused metric kind --- .../stats/BigtableStackdriverExportUtils.java | 12 +++--------- .../stats/BigtableStackdriverStatsExporter.java | 4 ++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 3dfbcaaf252e..b18df6d4a817 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -98,8 +98,8 @@ class BigtableStackdriverExportUtils { BuiltinMeasureConstants.ZONE.getName(), BuiltinMeasureConstants.TABLE.getName()); - private static final LabelKey clientIdLabelKey = - LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client id"); + private static final LabelKey CLIENT_UID_LABEL_KEY = + LabelKey.create(BuiltinMeasureConstants.CLIENT_UID.getName(), "client uid"); static com.google.monitoring.v3.TimeSeries convertTimeSeries( MetricDescriptor metricDescriptor, @@ -127,7 +127,7 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( metricTagValues.add(labelValues.get(i)); } } - metricTagKeys.add(clientIdLabelKey); + metricTagKeys.add(CLIENT_UID_LABEL_KEY); metricTagValues.add(LabelValue.create(clientId)); com.google.monitoring.v3.TimeSeries.Builder builder = @@ -162,9 +162,6 @@ static String getDefaultTaskValue() { private static MetricKind createMetricKind(Type type) { switch (type) { - case GAUGE_INT64: - case GAUGE_DOUBLE: - return MetricKind.GAUGE; case CUMULATIVE_DOUBLE: case CUMULATIVE_INT64: case CUMULATIVE_DISTRIBUTION: @@ -176,13 +173,10 @@ private static MetricKind createMetricKind(Type type) { private static com.google.api.MetricDescriptor.ValueType createValueType(Type type) { switch (type) { - case GAUGE_DOUBLE: case CUMULATIVE_DOUBLE: return com.google.api.MetricDescriptor.ValueType.DOUBLE; - case GAUGE_INT64: case CUMULATIVE_INT64: return com.google.api.MetricDescriptor.ValueType.INT64; - case GAUGE_DISTRIBUTION: case CUMULATIVE_DISTRIBUTION: return com.google.api.MetricDescriptor.ValueType.DISTRIBUTION; default: diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index d9d2ed94f159..f4f0985700e6 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -40,8 +40,8 @@ public class BigtableStackdriverStatsExporter { @GuardedBy("lock") private static BigtableStackdriverStatsExporter instance = null; - // Default export interval is 5 minutes - private static final Duration EXPORT_INTERVAL = Duration.create(60 * 5, 0); + // Default export interval is 1 minute + private static final Duration EXPORT_INTERVAL = Duration.create(60, 0); private static final String RESOURCE_TYPE = "bigtable_client_raw"; private final IntervalMetricReader intervalMetricReader; From ca6366049b2b979b3b34f518abeb7f113855df44 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 11 Aug 2022 11:38:10 -0700 Subject: [PATCH 26/30] get project id from the metrics --- .../BigtableCreateTimeSeriesExporter.java | 51 ++++++++++--------- .../stats/BigtableStackdriverExportUtils.java | 28 +++++++--- .../BigtableStackdriverStatsExporter.java | 8 ++- .../BigtableCreateTimeSeriesExporterTest.java | 1 - .../data/v2/BigtableDataSettings.java | 10 ++-- 5 files changed, 55 insertions(+), 43 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index 67550fb5120a..8997d0ce3c30 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -24,31 +24,28 @@ import io.opencensus.metrics.export.TimeSeries; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = Logger.getLogger(BigtableCreateTimeSeriesExporter.class.getName()); - - private final ProjectName projectName; private final MetricServiceClient metricServiceClient; private final MonitoredResource monitoredResource; private final String clientId; BigtableCreateTimeSeriesExporter( - String projectId, - MetricServiceClient metricServiceClient, - MonitoredResource monitoredResource) { - this.projectName = ProjectName.newBuilder().setProject(projectId).build(); + MetricServiceClient metricServiceClient, MonitoredResource monitoredResource) { this.metricServiceClient = metricServiceClient; this.monitoredResource = monitoredResource; this.clientId = BigtableStackdriverExportUtils.getDefaultTaskValue(); } public void export(Collection metrics) { - List timeSeriesList = new ArrayList(metrics.size()); + Map> projectToTimeSeries = new HashMap<>(); for (Metric metric : metrics) { // only export bigtable metrics @@ -56,23 +53,31 @@ public void export(Collection metrics) { continue; } - for (TimeSeries timeSeries : metric.getTimeSeriesList()) { - timeSeriesList.add( - BigtableStackdriverExportUtils.convertTimeSeries( - metric.getMetricDescriptor(), timeSeries, clientId, monitoredResource)); - } - } + try { + for (TimeSeries timeSeries : metric.getTimeSeriesList()) { + // Get the project id from the metrics so we could publish with multiple project ids + String projectId = + BigtableStackdriverExportUtils.getProjectId(metric.getMetricDescriptor(), timeSeries); + List timeSeriesList = + projectToTimeSeries.getOrDefault(projectId, new ArrayList<>()); + timeSeriesList.add( + BigtableStackdriverExportUtils.convertTimeSeries( + metric.getMetricDescriptor(), timeSeries, clientId, monitoredResource)); + projectToTimeSeries.put(projectId, timeSeriesList); + } - try { - CreateTimeSeriesRequest request = - CreateTimeSeriesRequest.newBuilder() - .setName(this.projectName.toString()) - .addAllTimeSeries(timeSeriesList) - .build(); - - this.metricServiceClient.createServiceTimeSeries(request); - } catch (Throwable e) { - logger.log(Level.WARNING, "Exception thrown when exporting TimeSeries.", e); + for (String projectId : projectToTimeSeries.keySet()) { + ProjectName projectName = ProjectName.of(projectId); + CreateTimeSeriesRequest request = + CreateTimeSeriesRequest.newBuilder() + .setName(projectName.toString()) + .addAllTimeSeries(projectToTimeSeries.get(projectId)) + .build(); + this.metricServiceClient.createServiceTimeSeries(request); + } + } catch (Throwable e) { + logger.log(Level.WARNING, "Exception thrown when exporting TimeSeries.", e); + } } } } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index b18df6d4a817..4d991ff4e0e4 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -137,27 +137,39 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( builder.setMetricKind(createMetricKind(metricType)); builder.setValueType(createValueType(metricType)); Timestamp startTimeStamp = timeSeries.getStartTimestamp(); - for (io.opencensus.metrics.export.Point point : timeSeries.getPoints()) { + for (Point point : timeSeries.getPoints()) { builder.addPoints(createPoint(point, startTimeStamp)); } return builder.build(); } + static String getProjectId(MetricDescriptor metricDescriptor, TimeSeries timeSeries) + throws Exception { + List labelKeys = metricDescriptor.getLabelKeys(); + List labelValues = timeSeries.getLabelValues(); + for (int i = 0; i < labelKeys.size(); i++) { + if (labelKeys.get(i).getKey().equals(BuiltinMeasureConstants.PROJECT_ID.getName())) { + return labelValues.get(i).getValue(); + } + } + throw new Exception("Can't find project id for the current timeseries"); + } + static String getDefaultTaskValue() { - String jvmName = ManagementFactory.getRuntimeMXBean().getName(); - if (jvmName.indexOf(64) < 1) { + // Something like '@' + final String jvmName = ManagementFactory.getRuntimeMXBean().getName(); + // If not the expected format then generate a random number. + if (jvmName.indexOf('@') < 1) { String hostname = "localhost"; - try { hostname = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { logger.log(Level.INFO, "Unable to get the hostname.", e); } - - return "java-" + (new SecureRandom()).nextInt() + "@" + hostname; - } else { - return "java-" + jvmName; + // Generate a random number and use the same format "random_number@hostname". + return "java-" + new SecureRandom().nextInt() + "@" + hostname; } + return "java-" + jvmName; } private static MetricKind createMetricKind(Type type) { diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index f4f0985700e6..03b1b6bfca57 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -47,7 +47,6 @@ public class BigtableStackdriverStatsExporter { private final IntervalMetricReader intervalMetricReader; private BigtableStackdriverStatsExporter( - String projectId, MetricServiceClient metricServiceClient, Duration exportInterval, MonitoredResource monitoredResource) { @@ -56,7 +55,7 @@ private BigtableStackdriverStatsExporter( intervalMetricReaderOptionsBuilder.setExportInterval(exportInterval); this.intervalMetricReader = IntervalMetricReader.create( - new BigtableCreateTimeSeriesExporter(projectId, metricServiceClient, monitoredResource), + new BigtableCreateTimeSeriesExporter(metricServiceClient, monitoredResource), MetricReader.create( MetricReader.Options.builder() .setMetricProducerManager( @@ -65,7 +64,7 @@ private BigtableStackdriverStatsExporter( intervalMetricReaderOptionsBuilder.build()); } - public static void register(Credentials credentials, String projectId) throws IOException { + public static void register(Credentials credentials) throws IOException { synchronized (lock) { Preconditions.checkState( instance == null, "Bigtable Stackdriver stats exporter is already created"); @@ -73,8 +72,7 @@ public static void register(Credentials credentials, String projectId) throws IO MetricServiceClient client = createMetricServiceClient(credentials, Duration.create(60L, 0)); MonitoredResource resourceType = MonitoredResource.newBuilder().setType(RESOURCE_TYPE).build(); - instance = - new BigtableStackdriverStatsExporter(projectId, client, EXPORT_INTERVAL, resourceType); + instance = new BigtableStackdriverStatsExporter(client, EXPORT_INTERVAL, resourceType); } } diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 6f97959e09f9..3f88bf399b04 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -67,7 +67,6 @@ public void setUp() { exporter = new BigtableCreateTimeSeriesExporter( - projectId, fakeMetricServiceClient, MonitoredResource.newBuilder().setType("bigtable-table").build()); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 8520a10e3b3e..83be6b24722e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -205,12 +205,11 @@ public static void enableGfeOpenCensusStats() { * your project allow listed for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 */ @BetaApi("Built in metric is not currently stable and may change in the future") - public static void registerBuiltinMetrics(String projectId) throws IOException { + public static void registerBuiltinMetrics() throws IOException { GoogleCredentials.getApplicationDefault(); if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { BuiltinViews.registerBigtableBuiltinViews(); - BigtableStackdriverStatsExporter.register( - GoogleCredentials.getApplicationDefault(), projectId); + BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault()); } } @@ -220,11 +219,10 @@ public static void registerBuiltinMetrics(String projectId) throws IOException { * https://forms.gle/xuhu6vCunn2MjV2m9 */ @BetaApi("Built in metric is not currently stable and may change in the future") - public static void registerBuiltinMetrics(Credentials credentials, String projectId) - throws IOException { + public static void registerBuiltinMetrics(Credentials credentials) throws IOException { if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { BuiltinViews.registerBigtableBuiltinViews(); - BigtableStackdriverStatsExporter.register(credentials, projectId); + BigtableStackdriverStatsExporter.register(credentials); } } From 5996119630b211dc5337f78026304c3981cb89c0 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 11 Aug 2022 11:43:47 -0700 Subject: [PATCH 27/30] clean up imports --- .../bigtable/stats/BigtableCreateTimeSeriesExporterTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java index 3f88bf399b04..26654c09afa5 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporterTest.java @@ -31,6 +31,7 @@ import io.opencensus.metrics.export.Metric; import io.opencensus.metrics.export.MetricDescriptor; import io.opencensus.metrics.export.Point; +import io.opencensus.metrics.export.TimeSeries; import io.opencensus.metrics.export.Value; import java.util.Arrays; import org.junit.After; @@ -99,7 +100,7 @@ public void testTimeSeries() { LabelKey.create(BuiltinMeasureConstants.ZONE.getName(), ""), LabelKey.create(BuiltinMeasureConstants.APP_PROFILE.getName(), ""))), Arrays.asList( - io.opencensus.metrics.export.TimeSeries.create( + TimeSeries.create( Arrays.asList( LabelValue.create(projectId), LabelValue.create(instanceId), From ee04276c0e4ea97d94eb03bfa800fb54c4fb8186 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Thu, 11 Aug 2022 15:00:13 -0700 Subject: [PATCH 28/30] remove unused method and rewrite create timeseries exporter --- .../BigtableCreateTimeSeriesExporter.java | 36 ++++++++++--------- .../stats/BigtableStackdriverExportUtils.java | 5 ++- .../BigtableStackdriverStatsExporter.java | 10 ------ .../data/v2/BigtableDataSettings.java | 33 +++++------------ .../v2/stub/metrics/BuiltinMetricsTracer.java | 4 +-- .../metrics/BuiltinMetricsTracerTest.java | 3 +- 6 files changed, 35 insertions(+), 56 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java index 8997d0ce3c30..ad2e76867cef 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableCreateTimeSeriesExporter.java @@ -21,14 +21,13 @@ import com.google.monitoring.v3.ProjectName; import io.opencensus.exporter.metrics.util.MetricExporter; import io.opencensus.metrics.export.Metric; -import io.opencensus.metrics.export.TimeSeries; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; final class BigtableCreateTimeSeriesExporter extends MetricExporter { private static final Logger logger = @@ -54,24 +53,29 @@ public void export(Collection metrics) { } try { - for (TimeSeries timeSeries : metric.getTimeSeriesList()) { - // Get the project id from the metrics so we could publish with multiple project ids - String projectId = - BigtableStackdriverExportUtils.getProjectId(metric.getMetricDescriptor(), timeSeries); - List timeSeriesList = - projectToTimeSeries.getOrDefault(projectId, new ArrayList<>()); - timeSeriesList.add( - BigtableStackdriverExportUtils.convertTimeSeries( - metric.getMetricDescriptor(), timeSeries, clientId, monitoredResource)); - projectToTimeSeries.put(projectId, timeSeriesList); - } + projectToTimeSeries = + metric.getTimeSeriesList().stream() + .collect( + Collectors.groupingBy( + timeSeries -> + BigtableStackdriverExportUtils.getProjectId( + metric.getMetricDescriptor(), timeSeries), + Collectors.mapping( + timeSeries -> + BigtableStackdriverExportUtils.convertTimeSeries( + metric.getMetricDescriptor(), + timeSeries, + clientId, + monitoredResource), + Collectors.toList()))); - for (String projectId : projectToTimeSeries.keySet()) { - ProjectName projectName = ProjectName.of(projectId); + for (Map.Entry> entry : + projectToTimeSeries.entrySet()) { + ProjectName projectName = ProjectName.of(entry.getKey()); CreateTimeSeriesRequest request = CreateTimeSeriesRequest.newBuilder() .setName(projectName.toString()) - .addAllTimeSeries(projectToTimeSeries.get(projectId)) + .addAllTimeSeries(entry.getValue()) .build(); this.metricServiceClient.createServiceTimeSeries(request); } diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java index 4d991ff4e0e4..bdca097050ac 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverExportUtils.java @@ -143,8 +143,7 @@ static com.google.monitoring.v3.TimeSeries convertTimeSeries( return builder.build(); } - static String getProjectId(MetricDescriptor metricDescriptor, TimeSeries timeSeries) - throws Exception { + static String getProjectId(MetricDescriptor metricDescriptor, TimeSeries timeSeries) { List labelKeys = metricDescriptor.getLabelKeys(); List labelValues = timeSeries.getLabelValues(); for (int i = 0; i < labelKeys.size(); i++) { @@ -152,7 +151,7 @@ static String getProjectId(MetricDescriptor metricDescriptor, TimeSeries timeSer return labelValues.get(i).getValue(); } } - throw new Exception("Can't find project id for the current timeseries"); + throw new IllegalStateException("Can't find project id for the current timeseries"); } static String getDefaultTaskValue() { diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java index 03b1b6bfca57..8896e52d8992 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BigtableStackdriverStatsExporter.java @@ -90,14 +90,4 @@ static MetricServiceClient createMetricServiceClient(Credentials credentials, Du settingsBuilder.createTimeSeriesSettings().setSimpleTimeoutNoRetries(stackdriverDuration); return MetricServiceClient.create(settingsBuilder.build()); } - - public static void unregister() { - synchronized (lock) { - if (instance != null) { - instance.intervalMetricReader.stop(); - } - - instance = null; - } - } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 83be6b24722e..be2594b00ff0 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -201,12 +201,13 @@ public static void enableGfeOpenCensusStats() { } /** - * Register built in metrics. This is an experimental feature. Please fill up this form to have - * your project allow listed for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 + * Register built in metrics. + * + *

This is an experimental feature. Please fill up this form to have your project allow listed + * for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 */ @BetaApi("Built in metric is not currently stable and may change in the future") public static void registerBuiltinMetrics() throws IOException { - GoogleCredentials.getApplicationDefault(); if (BUILTIN_METRICS_REGISTERED.compareAndSet(false, true)) { BuiltinViews.registerBigtableBuiltinViews(); BigtableStackdriverStatsExporter.register(GoogleCredentials.getApplicationDefault()); @@ -214,9 +215,11 @@ public static void registerBuiltinMetrics() throws IOException { } /** - * Register built in metrics with credentials. This is an experimental feature. Please fill up - * this form to have your project allow listed for the private preview: - * https://forms.gle/xuhu6vCunn2MjV2m9 + * Register built in metrics with credentials. The credentials need to have metric write access + * for all the projects you're publishing to. + * + *

This is an experimental feature. Please fill up this form to have your project allow listed + * for the private preview: https://forms.gle/xuhu6vCunn2MjV2m9 */ @BetaApi("Built in metric is not currently stable and may change in the future") public static void registerBuiltinMetrics(Credentials credentials) throws IOException { @@ -226,14 +229,6 @@ public static void registerBuiltinMetrics(Credentials credentials) throws IOExce } } - /** Unregister built in metrics. * */ - @BetaApi("Built in metrics is not currently stable and may change in the future") - public static void unregisterBuiltinMetrics() { - if (BUILTIN_METRICS_REGISTERED.compareAndSet(true, false)) { - BigtableStackdriverStatsExporter.unregister(); - } - } - /** Returns the target project id. */ public String getProjectId() { return stubSettings.getProjectId(); @@ -283,16 +278,6 @@ public Long getBatchMutationsTargetRpcLatencyMs() { return stubSettings.bulkMutateRowsSettings().getTargetRpcLatencyMs(); } - /** - * Gets if built in metrics are registered. - * - * @return - */ - @BetaApi("Built in metrics is not currently stable and may change in the future") - public static boolean isBuiltinMetricsRegistered() { - return BUILTIN_METRICS_REGISTERED.get(); - } - /** Returns the underlying RPC settings. */ public EnhancedBigtableStubSettings getStubSettings() { return stubSettings; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java index 1c77a9c32369..0d42ba806dc6 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java @@ -66,9 +66,9 @@ class BuiltinMetricsTracer extends BigtableTracer { private AtomicInteger requestLeft = new AtomicInteger(0); // Monitored resource labels - private String tableId = "undefined"; + private String tableId = "unspecified"; private String zone = "global"; - private String cluster = "global"; + private String cluster = "unspecified"; // gfe stats private AtomicLong gfeMissingHeaders = new AtomicLong(0); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index 2989e5c84a24..7f9e7481cb5d 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -340,7 +340,8 @@ public void testMutateRowAttempts() { verify(statsRecorderWrapper, timeout(50).times(fakeService.getAttemptCounter().get() + 1)) .record(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); assertThat(zone.getAllValues()).containsExactly("global", "global", ZONE, ZONE); - assertThat(cluster.getAllValues()).containsExactly("global", "global", CLUSTER, CLUSTER); + assertThat(cluster.getAllValues()) + .containsExactly("unspecified", "unspecified", CLUSTER, CLUSTER); assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "UNAVAILABLE", "OK", "OK"); } From 7607480a1133e4fb1acb91368b45f6d9cefb62a8 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Fri, 12 Aug 2022 07:32:51 -0700 Subject: [PATCH 29/30] fix integration test --- .../cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java | 4 ++-- .../cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java index 2f9c8ff639f3..b0e12d5ade73 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/StreamingMetricsMetadataIT.java @@ -84,7 +84,7 @@ public void testFailure() throws InterruptedException { Thread.sleep(100); List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings(); - assertThat(tagValueStrings).contains("undefined"); - assertThat(tagValueStrings).contains("undefined"); + assertThat(tagValueStrings).contains("unspecified"); + assertThat(tagValueStrings).contains("global"); } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java index 9fd132ed899a..0c8570b26113 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/UnaryMetricsMetadataIT.java @@ -94,7 +94,7 @@ public void testFailure() throws InterruptedException { Thread.sleep(100); List tagValueStrings = StatsWrapper.getOperationLatencyViewTagValueStrings(); - assertThat(tagValueStrings).contains("undefined"); - assertThat(tagValueStrings).contains("undefined"); + assertThat(tagValueStrings).contains("unspecified"); + assertThat(tagValueStrings).contains("global"); } } From 559a168d88c4f5124b9c863a2d26e0a42cea478f Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 12 Aug 2022 15:37:40 +0000 Subject: [PATCH 30/30] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 61404dff4a90..dd8140a57f8d 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigtable' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigtable:2.10.1' +implementation 'com.google.cloud:google-cloud-bigtable:2.10.3' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.10.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.10.3" ``` ## Authentication