diff --git a/bigtable-hbase/pom.xml b/bigtable-hbase/pom.xml index 377cdc4f9e..22144f3b19 100644 --- a/bigtable-hbase/pom.xml +++ b/bigtable-hbase/pom.xml @@ -251,6 +251,7 @@ **/TestBatch.java **/TestBufferedMutator.java **/TestCheckAndMutate.java + **/TestColumnFamilyAdmin.java **/TestCreateTable.java **/TestDisableTable.java **/TestDelete.java diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BatchExecutor.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BatchExecutor.java index cf418f24bd..ae4ce0240f 100644 --- a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BatchExecutor.java +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BatchExecutor.java @@ -10,6 +10,7 @@ import com.google.cloud.bigtable.hbase.adapters.OperationAdapter; import com.google.cloud.bigtable.hbase.adapters.ResponseAdapter; import com.google.cloud.bigtable.hbase.adapters.RowMutationsAdapter; +import com.google.cloud.bigtable.hbase.adapters.TableMetadataSetter; import com.google.cloud.hadoop.hbase.BigtableClient; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutator.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutator.java index 220d31afcc..1518d0d423 100644 --- a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutator.java +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutator.java @@ -25,6 +25,7 @@ import com.google.cloud.bigtable.hbase.adapters.ResponseAdapter; import com.google.cloud.bigtable.hbase.adapters.RowMutationsAdapter; import com.google.cloud.bigtable.hbase.adapters.ScanAdapter; +import com.google.cloud.bigtable.hbase.adapters.TableMetadataSetter; import com.google.cloud.bigtable.hbase.adapters.UnsupportedOperationAdapter; import com.google.cloud.hadoop.hbase.BigtableClient; import com.google.common.collect.Lists; diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableRegionLocator.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableRegionLocator.java index ad7531d1b1..1603c6e11a 100644 --- a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableRegionLocator.java +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableRegionLocator.java @@ -3,6 +3,7 @@ import com.google.bigtable.v1.SampleRowKeysRequest; import com.google.bigtable.v1.SampleRowKeysResponse; import com.google.cloud.bigtable.hbase.adapters.SampledRowKeysAdapter; +import com.google.cloud.bigtable.hbase.adapters.TableMetadataSetter; import com.google.cloud.hadoop.hbase.BigtableClient; import com.google.common.collect.ImmutableList; diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableTable.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableTable.java index 47550c7492..e34b4e51b9 100644 --- a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableTable.java +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableTable.java @@ -13,8 +13,8 @@ */ package com.google.cloud.bigtable.hbase; -import com.google.bigtable.v1.CheckAndMutateRowResponse; import com.google.bigtable.v1.CheckAndMutateRowRequest; +import com.google.bigtable.v1.CheckAndMutateRowResponse; import com.google.bigtable.v1.MutateRowRequest; import com.google.bigtable.v1.ReadModifyWriteRowRequest; import com.google.bigtable.v1.ReadRowsRequest; @@ -30,6 +30,7 @@ import com.google.cloud.bigtable.hbase.adapters.ResponseAdapter; import com.google.cloud.bigtable.hbase.adapters.RowMutationsAdapter; import com.google.cloud.bigtable.hbase.adapters.ScanAdapter; +import com.google.cloud.bigtable.hbase.adapters.TableMetadataSetter; import com.google.cloud.bigtable.hbase.adapters.UnsupportedOperationAdapter; import com.google.cloud.hadoop.hbase.BigtableClient; import com.google.common.util.concurrent.ListeningExecutorService; diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ClusterMetadataSetter.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ClusterMetadataSetter.java new file mode 100644 index 0000000000..ddda549a88 --- /dev/null +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ClusterMetadataSetter.java @@ -0,0 +1,50 @@ + package com.google.cloud.bigtable.hbase.adapters; + +import com.google.bigtable.admin.table.v1.CreateTableRequest; +import com.google.bigtable.admin.table.v1.ListTablesRequest; +import com.google.cloud.bigtable.hbase.BigtableOptions; +import com.google.common.base.Preconditions; + +/** + * Utility class that will set given project and cluster name within service messages. + */ +public class ClusterMetadataSetter { + public static final String BIGTABLE_V1_CLUSTER_FMT = + "projects/%s/zones/%s/clusters/%s"; + + public static ClusterMetadataSetter from(BigtableOptions options) { + return new ClusterMetadataSetter( + options.getProjectId(), options.getZone(), options.getCluster()); + } + + private final String formattedV1ClusterName; + + public ClusterMetadataSetter(String projectId, String zone, String clusterName) { + this.formattedV1ClusterName = + String.format(BIGTABLE_V1_CLUSTER_FMT, projectId, zone, clusterName); + } + + public void setMetadata(ListTablesRequest.Builder builder) { + builder.setName(formattedV1ClusterName); + } + + public void setMetadata(CreateTableRequest.Builder builder) { + builder.setName(formattedV1ClusterName); + } + + public String getFormattedV1ClusterName() { + return formattedV1ClusterName; + } + + public String toHBaseTableName(String fullyQualifiedBigtableName) { + Preconditions.checkNotNull(fullyQualifiedBigtableName, "table name cannot be null"); + String tablesPrefix = + String.format("%s/%s/", formattedV1ClusterName, TableMetadataSetter.TABLE_SEPARATOR); + Preconditions.checkState(fullyQualifiedBigtableName.startsWith(tablesPrefix), + "'%s' does not start with '%s'", fullyQualifiedBigtableName, tablesPrefix); + String tableName = fullyQualifiedBigtableName.substring(tablesPrefix.length()); + Preconditions.checkState(tableName.length() > 0, "TableName is blank"); + return tableName; + } + +} diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnDescriptorAdapter.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnDescriptorAdapter.java index 247568e80e..99fc9aa2fd 100644 --- a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnDescriptorAdapter.java +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnDescriptorAdapter.java @@ -1,10 +1,11 @@ package com.google.cloud.bigtable.hbase.adapters; import com.google.api.client.util.Strings; -import com.google.bigtable.anviltop.AnviltopData; +import com.google.bigtable.admin.table.v1.ColumnFamily; import com.google.cloud.bigtable.hbase.BigtableConstants; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -20,13 +21,13 @@ /** * Adapt a single instance of an HBase {@link HColumnDescriptor} to - * an instance of {@link AnviltopData.ColumnFamily} + * an instance of {@link ColumnFamily} */ public class ColumnDescriptorAdapter { /** * Configuration keys that we can support unconditionally and we provide - * a mapped version of the column descriptor to bigtable. + * a mapped version of the column descriptor to Bigtable. */ public static final Set SUPPORTED_OPTION_KEYS = ImmutableSet.of( @@ -130,7 +131,7 @@ public static void throwIfRequestingUnsupportedFeatures(HColumnDescriptor column } /** - * Construct an bigtable GC expression from the given column descriptor. + * Construct an Bigtable GC expression from the given column descriptor. */ public static String buildGarbageCollectionExpression(HColumnDescriptor columnDescriptor) { int maxVersions = columnDescriptor.getMaxVersions(); @@ -159,22 +160,106 @@ public static String buildGarbageCollectionExpression(HColumnDescriptor columnDe return buffer.toString(); } + static Splitter gcExpressionOrSplitter = Splitter.on("|").trimResults().omitEmptyStrings(); + static Splitter gcExpressionAndSplitter = Splitter.on("&").trimResults().omitEmptyStrings(); + /** - * Adapt a single instance of an HBase {@link HColumnDescriptor} to - * an instance of {@link AnviltopData.ColumnFamily} + * Parse a Bigtable GC-Expression that is in line with + * {@link #buildGarbageCollectionExpression(HColumnDescriptor)} into a the provided + * {@link HColumnDescriptor}. This method will likely throw IllegalStateException if the + * GC Expression isn't similar to buildGarbageCollectionExpression's expression. */ - public AnviltopData.ColumnFamily.Builder adapt(HColumnDescriptor columnDescriptor) { + public static void convertGarbageCollectionExpression(String gcExpression, + HColumnDescriptor columnDescriptor) { + String maxVersionExpression = null; + String minVersionExpression = null; + String ttlExpression = null; + if (gcExpression.contains("||")) { + for (String expression : gcExpressionOrSplitter.split(gcExpression)) { + if (expression.contains("age()")) { + for (String expressionComponent : gcExpressionAndSplitter.split(expression)) { + if (expressionComponent.contains("age()") + && expressionComponent.contains(">")) { + ttlExpression = expressionComponent.replaceAll("[()]", ""); + } else if (expressionComponent.contains("version()") + && expressionComponent.contains(">")) { + minVersionExpression = expressionComponent.replaceAll("[()]", ""); + } else { + throw new IllegalStateException(String.format( + "Expression: '%s' could not be parsed.", expression)); + } + } + } else if (expression.contains("version()") && expression.contains(">")) { + maxVersionExpression = expression.replaceAll("[()]", ""); + } else { + throw new IllegalStateException(String.format( + "Expression: '%s' could not be parsed.", expression)); + } + } + } else if (gcExpression.contains("version()") && gcExpression.contains(">")) { + maxVersionExpression = gcExpression.replaceAll("[()]", ""); + } else { + throw new IllegalStateException(String.format( + "Expression: '%s' could not be parsed.", maxVersionExpression)); + } + + int maxVersions = getInteger(maxVersionExpression); + columnDescriptor.setMaxVersions(maxVersions); + + if (minVersionExpression != null) { + int minVersions = getInteger(minVersionExpression); + Preconditions.checkState(minVersions < maxVersions, + "HColumnDescriptor min versions must be less than max versions."); + columnDescriptor.setMinVersions(minVersions); + } + if (ttlExpression != null) { + long bigtableTtl = getLong(ttlExpression); + int ttlSeconds = + (int) TimeUnit.SECONDS.convert(bigtableTtl, BigtableConstants.BIGTABLE_TIMEUNIT); + if (ttlSeconds != HColumnDescriptor.DEFAULT_TTL) { + columnDescriptor.setTimeToLive(ttlSeconds); + } + } + } + + private static Integer getInteger(String expression) { + return Integer.valueOf(getNumber(expression)); + } + + private static Long getLong(String expression) { + return Long.valueOf(getNumber(expression)); + } + + public static String getNumber(String expression) { + return expression.replaceFirst(".*> *(\\d+).*", "$1"); + } + + /** + *

Adapt a single instance of an HBase {@link HColumnDescriptor} to + * an instance of {@link com.google.bigtable.admin.table.v1.ColumnFamily.Builder}.

+ * + *

NOTE: This method does not set the name of the ColumnFamily.Builder. The assumption is + * that the CreateTableRequest or CreateColumFamilyRequest takes care of the naming. As of now + * (3/11/2015), the server insists on having a blank name.

+ */ + public ColumnFamily.Builder adapt(HColumnDescriptor columnDescriptor) { throwIfRequestingUnknownFeatures(columnDescriptor); throwIfRequestingUnsupportedFeatures(columnDescriptor); - AnviltopData.ColumnFamily.Builder resultBuilder = AnviltopData.ColumnFamily.newBuilder(); - - resultBuilder.setName(columnDescriptor.getNameAsString()); + ColumnFamily.Builder resultBuilder = ColumnFamily.newBuilder(); String gcExpression = buildGarbageCollectionExpression(columnDescriptor); if (!Strings.isNullOrEmpty(gcExpression)) { resultBuilder.setGcExpression(gcExpression); } - return resultBuilder; } + + public HColumnDescriptor adapt(String familyName, ColumnFamily columnFamily) { + HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(familyName); + String gcExpression = columnFamily.getGcExpression(); + if (gcExpression != null) { + convertGarbageCollectionExpression(gcExpression, hColumnDescriptor); + } + return hColumnDescriptor; + } } diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnFamilyFormatter.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnFamilyFormatter.java new file mode 100644 index 0000000000..e110a99513 --- /dev/null +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/ColumnFamilyFormatter.java @@ -0,0 +1,34 @@ +package com.google.cloud.bigtable.hbase.adapters; + +import org.apache.hadoop.hbase.TableName; + +import com.google.cloud.bigtable.hbase.BigtableOptions; +import com.google.common.base.Preconditions; + +public class ColumnFamilyFormatter { + + final static String COLUMN_FAMILIES_SEPARARATOR = "columnFamilies"; + + public static ColumnFamilyFormatter from(TableName tableName, BigtableOptions options) { + return new ColumnFamilyFormatter(TableMetadataSetter.from(tableName, options)); + } + + private final TableMetadataSetter tableMetadataSetter; + + public ColumnFamilyFormatter(TableMetadataSetter tableMetadataSetter) { + this.tableMetadataSetter = tableMetadataSetter; + } + + public String formatForBigtable(String hbaseColumnFamily) { + return String.format("%s/%s/%s", tableMetadataSetter.getFormattedV1TableName(), + COLUMN_FAMILIES_SEPARARATOR, hbaseColumnFamily); + } + + public String formatForHBase(String bigtableColumnFamily) { + String tableName = tableMetadataSetter.getFormattedV1TableName(); + String familyPrefix = String.format("%s/%s", tableName, COLUMN_FAMILIES_SEPARARATOR); + Preconditions.checkState(bigtableColumnFamily.startsWith(familyPrefix), + "Column family '%s' needs to start with '%s'", bigtableColumnFamily, tableName); + return bigtableColumnFamily.substring(familyPrefix.length()); + } +} diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/TableAdapter.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/TableAdapter.java new file mode 100644 index 0000000000..6b16fd64ea --- /dev/null +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/TableAdapter.java @@ -0,0 +1,43 @@ +package com.google.cloud.bigtable.hbase.adapters; + +import com.google.bigtable.admin.table.v1.ColumnFamily; +import com.google.bigtable.admin.table.v1.Table; +import com.google.cloud.bigtable.hbase.BigtableOptions; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; + +import java.util.Map; +import java.util.Map.Entry; + +public class TableAdapter { + + private final ClusterMetadataSetter clusterMetadataSetter; + private final ColumnDescriptorAdapter columnDescriptorAdapter; + + public TableAdapter(BigtableOptions options, ColumnDescriptorAdapter columnDescriptorAdapter) { + this.clusterMetadataSetter = ClusterMetadataSetter.from(options); + this.columnDescriptorAdapter = columnDescriptorAdapter; + } + + public Table adapt(HTableDescriptor desc) { + Table.Builder tableBuilder = Table.newBuilder(); + Map columnFamilies = tableBuilder.getMutableColumnFamilies(); + for (HColumnDescriptor column : desc.getColumnFamilies()) { + ColumnFamily columnFamily = columnDescriptorAdapter.adapt(column).build(); + String columnName = column.getNameAsString(); + columnFamilies.put(columnName, columnFamily); + } + return tableBuilder.build(); + } + + public HTableDescriptor adapt(Table table) { + String tableNameStr = clusterMetadataSetter.toHBaseTableName(table.getName()); + HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableNameStr)); + for (Entry entry : table.getColumnFamilies().entrySet()) { + tableDescriptor.addFamily(columnDescriptorAdapter.adapt(entry.getKey(), entry.getValue())); + } + return tableDescriptor; + } +} diff --git a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/TableMetadataSetter.java b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/TableMetadataSetter.java similarity index 64% rename from bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/TableMetadataSetter.java rename to bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/TableMetadataSetter.java index ceffde4963..db9f6346cf 100644 --- a/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/TableMetadataSetter.java +++ b/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/TableMetadataSetter.java @@ -1,10 +1,14 @@ -package com.google.cloud.bigtable.hbase; +package com.google.cloud.bigtable.hbase.adapters; +import com.google.bigtable.admin.table.v1.CreateColumnFamilyRequest; +import com.google.bigtable.admin.table.v1.DeleteTableRequest; +import com.google.bigtable.admin.table.v1.Table; import com.google.bigtable.v1.CheckAndMutateRowRequest; import com.google.bigtable.v1.MutateRowRequest; import com.google.bigtable.v1.ReadModifyWriteRowRequest; -import com.google.bigtable.v1.SampleRowKeysRequest; import com.google.bigtable.v1.ReadRowsRequest; +import com.google.bigtable.v1.SampleRowKeysRequest; +import com.google.cloud.bigtable.hbase.BigtableOptions; import org.apache.hadoop.hbase.TableName; @@ -13,8 +17,9 @@ * service messages. */ public class TableMetadataSetter { + public static final String TABLE_SEPARATOR = "tables"; public static final String BIGTABLE_V1_TABLENAME_FMT = - "projects/%s/zones/%s/clusters/%s/tables/%s"; + ClusterMetadataSetter.BIGTABLE_V1_CLUSTER_FMT + "/" + TABLE_SEPARATOR + "/%s"; public static TableMetadataSetter from(TableName tableName, BigtableOptions options) { return new TableMetadataSetter( @@ -29,6 +34,21 @@ public TableMetadataSetter( projectId, zone, clusterName, tableName.getQualifierAsString()); } + // Table Admin builders + public void setMetadata(Table.Builder builder) { + builder.setName(formattedV1TableName); + } + + public void setMetadata(DeleteTableRequest.Builder builder) { + builder.setName(formattedV1TableName); + } + + public void setMetadata(CreateColumnFamilyRequest.Builder builder) { + builder.setName(formattedV1TableName); + } + + // Table read/write builders. + public void setMetadata(MutateRowRequest.Builder builder) { builder.setTableName(formattedV1TableName); } @@ -48,4 +68,8 @@ public void setMetadata(SampleRowKeysRequest.Builder builder) { public void setMetadata(ReadRowsRequest.Builder builder) { builder.setTableName(formattedV1TableName); } + + public String getFormattedV1TableName() { + return formattedV1TableName; + } } diff --git a/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableAdmin.java b/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableAdmin.java index 005c6a04b1..0f602d8636 100644 --- a/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableAdmin.java +++ b/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableAdmin.java @@ -37,22 +37,27 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; -import com.google.bigtable.anviltop.AnviltopAdminServiceMessages.CreateFamilyRequest; -import com.google.bigtable.anviltop.AnviltopAdminServiceMessages.CreateTableRequest; -import com.google.bigtable.anviltop.AnviltopAdminServiceMessages.DeleteFamilyRequest; -import com.google.bigtable.anviltop.AnviltopAdminServiceMessages.DeleteTableRequest; -import com.google.bigtable.anviltop.AnviltopAdminServiceMessages.ListTablesRequest; -import com.google.bigtable.anviltop.AnviltopAdminServiceMessages.ListTablesResponse; +import com.google.bigtable.admin.table.v1.ColumnFamily; +import com.google.bigtable.admin.table.v1.CreateColumnFamilyRequest; +import com.google.bigtable.admin.table.v1.CreateTableRequest; +import com.google.bigtable.admin.table.v1.DeleteColumnFamilyRequest; +import com.google.bigtable.admin.table.v1.DeleteTableRequest; +import com.google.bigtable.admin.table.v1.DeleteTableRequest.Builder; +import com.google.bigtable.admin.table.v1.ListTablesRequest; +import com.google.bigtable.admin.table.v1.ListTablesResponse; import com.google.cloud.bigtable.hbase.BigtableOptions; import com.google.cloud.bigtable.hbase.Logger; +import com.google.cloud.bigtable.hbase.adapters.ClusterMetadataSetter; import com.google.cloud.bigtable.hbase.adapters.ColumnDescriptorAdapter; -import com.google.cloud.hadoop.hbase.AnviltopAdminClient; -import com.google.protobuf.ByteString; +import com.google.cloud.bigtable.hbase.adapters.ColumnFamilyFormatter; +import com.google.cloud.bigtable.hbase.adapters.TableAdapter; +import com.google.cloud.bigtable.hbase.adapters.TableMetadataSetter; +import com.google.cloud.hadoop.hbase.BigtableAdminClient; public class BigtableAdmin implements Admin { private static final Logger LOG = new Logger(BigtableAdmin.class); - + /** * Bigtable doesn't require disabling tables before deletes or schema changes. Some clients do * call disable first, and then check for disable before deletes or schema changes. We're keeping @@ -64,14 +69,17 @@ public class BigtableAdmin implements Admin { private final Configuration configuration; private final BigtableOptions options; private final BigtableConnection connection; - private final AnviltopAdminClient bigtableAdminClient; + private final BigtableAdminClient bigtableAdminClient; + + private ClusterMetadataSetter clusterMetadataSetter; private final ColumnDescriptorAdapter columnDescriptorAdapter = new ColumnDescriptorAdapter(); + private final TableAdapter tableAdapter; public BigtableAdmin( BigtableOptions options, Configuration configuration, BigtableConnection connection, - AnviltopAdminClient bigtableAdminClient, + BigtableAdminClient bigtableAdminClient, Set disabledTables) { LOG.debug("Creating BigtableAdmin"); this.configuration = configuration; @@ -79,6 +87,8 @@ public BigtableAdmin( this.connection = connection; this.bigtableAdminClient = bigtableAdminClient; this.disabledTables = disabledTables; + this.clusterMetadataSetter = ClusterMetadataSetter.from(options); + this.tableAdapter = new TableAdapter(options, columnDescriptorAdapter); } @Override @@ -125,8 +135,9 @@ public HTableDescriptor[] listTables() throws IOException { @Override public HTableDescriptor[] listTables(Pattern pattern) throws IOException { + // NOTE: We don't have systables. ListTablesRequest.Builder builder = ListTablesRequest.newBuilder(); - builder.setProjectId(options.getProjectId()); + clusterMetadataSetter.setMetadata(builder); ListTablesResponse response; try { response = bigtableAdminClient.listTables(builder.build()); @@ -134,9 +145,12 @@ public HTableDescriptor[] listTables(Pattern pattern) throws IOException { throw new IOException("Failed to listTables", throwable); } List result = new ArrayList<>(); - for (String tableName : response.getTableNameList()) { - if (pattern.matcher(tableName).matches()) { - result.add(new HTableDescriptor(TableName.valueOf(tableName))); + for (com.google.bigtable.admin.table.v1.Table table : response.getTablesList()) { + // Convert the fully qualified Bigtable table name into an HBase table name before + // comparing against the pattern. + HTableDescriptor tableDescriptor = tableAdapter.adapt(table); + if (pattern.matcher(tableDescriptor.getTableName().getNameAsString()).matches()) { + result.add(tableDescriptor); } } @@ -146,7 +160,6 @@ public HTableDescriptor[] listTables(Pattern pattern) throws IOException { @Override public HTableDescriptor[] listTables(final Pattern pattern, final boolean includeSysTables) throws IOException { - // We don't have systables. return listTables(pattern); } @@ -160,17 +173,15 @@ public TableName[] listTableNames(String patternStr) throws IOException { @Override public TableName[] listTableNames(Pattern pattern) throws IOException { - ListTablesRequest.Builder builder = ListTablesRequest.newBuilder(); - builder.setProjectId(options.getProjectId()); - ListTablesResponse response = bigtableAdminClient.listTables(builder.build()); - List result = new ArrayList<>(); - for (String tableName : response.getTableNameList()) { - if (pattern.matcher(tableName).matches()) { - result.add(TableName.valueOf(tableName)); - } - } + return getTableNames(listTables(pattern)); + } - return result.toArray(new TableName[result.size()]); + private TableName[] getTableNames(HTableDescriptor[] tableList) { + TableName[] tableNames = new TableName[tableList.length]; + for (int i=0; i 0) { + return response[0]; + } else { + throw new TableNotFoundException(tableName.getNameAsString()); + } } // Used by the Hbase shell but not defined by Admin. Will be removed once the // shell is switch to use the methods defined in the interface. @Deprecated public String[] getTableNames(String regex) throws IOException { - HTableDescriptor[] tableDescriptors = listTables(); + HTableDescriptor[] tableDescriptors = listTables(regex); String[] tableNames = new String[tableDescriptors.length]; - int i = 0; - for (HTableDescriptor tableDescriptor : tableDescriptors) { - tableNames[i] = tableDescriptor.getNameAsString(); - i++; + for (int i = 0; i < tableDescriptors.length; i++) { + tableNames[i] = tableDescriptors[i].getNameAsString(); } return tableNames; } @@ -228,12 +239,9 @@ public String[] getTableNames(String regex) throws IOException { @Override public void createTable(HTableDescriptor desc) throws IOException { CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); - builder.setProjectId(options.getProjectId()).setTableNameBytes( - ByteString.copyFrom(desc.getTableName().getName())); - - for (HColumnDescriptor column : desc.getColumnFamilies()) { - builder.addColumnFamilies(columnDescriptorAdapter.adapt(column)); - } + clusterMetadataSetter.setMetadata(builder); + builder.setTableId(desc.getTableName().getQualifierAsString()); + builder.setTable(tableAdapter.adapt(desc)); try { bigtableAdminClient.createTable(builder.build()); } catch (Throwable throwable) { @@ -263,11 +271,10 @@ public void createTableAsync(HTableDescriptor desc, byte[][] splitKeys) throws I @Override public void deleteTable(TableName tableName) throws IOException { + Builder deleteBuilder = DeleteTableRequest.newBuilder(); + TableMetadataSetter.from(tableName, options).setMetadata(deleteBuilder); try { - bigtableAdminClient.deleteTable(DeleteTableRequest.newBuilder() - .setProjectId(options.getProjectId()) - .setTableNameBytes(ByteString.copyFrom(tableName.getQualifier())) - .build()); + bigtableAdminClient.deleteTable(deleteBuilder.build()); } catch (Throwable throwable) { throw new IOException( String.format( @@ -421,13 +428,20 @@ public Pair getAlterStatus(byte[] tableName) throws IOExceptio @Override public void addColumn(TableName tableName, HColumnDescriptor column) throws IOException { + TableMetadataSetter tableMetadataSetter = TableMetadataSetter.from(tableName, options); + + ColumnFamily.Builder columnFamily = + columnDescriptorAdapter.adapt(column); + + CreateColumnFamilyRequest.Builder createColumnFamilyBuilder = + CreateColumnFamilyRequest.newBuilder() + .setColumnFamilyId(column.getNameAsString()) + .setColumnFamily(columnFamily); + + tableMetadataSetter.setMetadata(createColumnFamilyBuilder); + try { - bigtableAdminClient.createFamily( - CreateFamilyRequest.newBuilder() - .setProjectId(options.getProjectId()) - .setTableName(tableName.getQualifierAsString()) - .setFamily(columnDescriptorAdapter.adapt(column)) - .build()); + bigtableAdminClient.createColumnFamily(createColumnFamilyBuilder.build()); } catch (Throwable throwable) { throw new IOException( String.format( @@ -447,12 +461,11 @@ public void addColumn(String tableName, HColumnDescriptor column) throws IOExcep @Override public void deleteColumn(TableName tableName, byte[] columnName) throws IOException { + ColumnFamilyFormatter formatter = ColumnFamilyFormatter.from(tableName, options); + String bigtableColumnName = formatter.formatForBigtable(Bytes.toString(columnName)); try { - bigtableAdminClient.deleteFamily( - DeleteFamilyRequest.newBuilder() - .setProjectId(options.getProjectId()) - .setTableName(tableName.getQualifierAsString()) - .setFamilyNameBytes(ByteString.copyFrom(columnName)).build()); + bigtableAdminClient.deleteColumnFamily( + DeleteColumnFamilyRequest.newBuilder().setName(bigtableColumnName).build()); } catch (Throwable throwable) { throw new IOException( String.format( diff --git a/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableConnection.java b/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableConnection.java index b6db436672..ec03a460ab 100644 --- a/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableConnection.java +++ b/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/BigtableConnection.java @@ -37,8 +37,8 @@ import com.google.cloud.bigtable.hbase.BigtableRegionLocator; import com.google.cloud.bigtable.hbase.BigtableTable; import com.google.cloud.bigtable.hbase.Logger; -import com.google.cloud.hadoop.hbase.AnviltopAdminBlockingGrpcClient; -import com.google.cloud.hadoop.hbase.AnviltopAdminClient; +import com.google.cloud.hadoop.hbase.BigtableAdminClient; +import com.google.cloud.hadoop.hbase.BigtableAdminGrpcClient; import com.google.cloud.hadoop.hbase.BigtableClient; import com.google.cloud.hadoop.hbase.BigtableGrpcClient; import com.google.cloud.hadoop.hbase.ChannelOptions; @@ -55,7 +55,7 @@ public class BigtableConnection implements Connection, Closeable { private volatile boolean aborted; private volatile ExecutorService batchPool = null; private BigtableClient client; - private AnviltopAdminClient bigtableAdminClient; + private BigtableAdminClient bigtableAdminClient; private volatile boolean cleanupPool = false; private final BigtableOptions options; @@ -106,12 +106,12 @@ public BigtableConnection(Configuration conf) throws IOException { this.tableConfig = new TableConfiguration(conf); } - private AnviltopAdminClient getAdminClient( + private BigtableAdminClient getAdminClient( TransportOptions transportOptions, ChannelOptions channelOptions, ExecutorService executorService) { - return AnviltopAdminBlockingGrpcClient.createClient( + return BigtableAdminGrpcClient.createClient( transportOptions, channelOptions, executorService); } @@ -201,7 +201,7 @@ public RegionLocator getRegionLocator(TableName tableName) throws IOException { @Override public Admin getAdmin() throws IOException { - return new BigtableAdmin(options, conf, this, bigtableAdminClient, this.disabledTables ); + return new BigtableAdmin(options, conf, this, bigtableAdminClient, this.disabledTables); } @Override diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/AbstractTest.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/AbstractTest.java index 5ac4157f49..b2fc8e5d03 100644 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/AbstractTest.java +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/AbstractTest.java @@ -53,10 +53,16 @@ public Statement apply(Statement base, Description description) { @Override protected void before() throws Throwable { connection = createNewConnection(); + onConnectionCreated(); } @Override protected void after() { + try { + preConnectionClose(); + } catch (IOException e) { + new Logger(AbstractTest.this.getClass()).warn("Could not perform preClose", e); + } try { connection.close(); } catch (IOException e) { @@ -71,6 +77,16 @@ public Connection createNewConnection() throws IOException { return ConnectionFactory.createConnection(conf); } + /** Hook to setup class level resources after the connection is created. */ + @SuppressWarnings("unused") + protected void onConnectionCreated() throws IOException { + } + + /** Hook to remove class level resources after the connection is created. */ + @SuppressWarnings("unused") + protected void preConnectionClose() throws IOException { + } + protected static class QualifierValue implements Comparable { protected final byte[] qualifier; protected final byte[] value; diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/IntegrationTests.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/IntegrationTests.java index b698d8c247..bb2139f251 100644 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/IntegrationTests.java +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/IntegrationTests.java @@ -33,6 +33,7 @@ TestBatch.class, TestBufferedMutator.class, TestCheckAndMutate.class, + TestColumnFamilyAdmin.class, TestCreateTable.class, TestDisableTable.class, TestDelete.class, diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestColumnFamilyAdmin.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestColumnFamilyAdmin.java new file mode 100644 index 0000000000..189a3caa55 --- /dev/null +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestColumnFamilyAdmin.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2013 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.bigtable.hbase; + +import static com.google.cloud.bigtable.hbase.IntegrationTests.COLUMN_FAMILY; + +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +/** + * Tests creation and deletion of column families. + */ +public class TestColumnFamilyAdmin extends AbstractTest { + + private static byte[] DELETE_COLUMN_FAMILY = + Bytes.toBytes(Bytes.toString(COLUMN_FAMILY) + "_remove"); + + private Admin admin; + private TableName tableName; + private HTableDescriptor descriptor; + + @Override + protected void onConnectionCreated() throws IOException { + admin = connection.getAdmin(); + tableName = IntegrationTests.newTestTableName(); + + descriptor = new HTableDescriptor(tableName); + descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); + descriptor.addFamily(new HColumnDescriptor(DELETE_COLUMN_FAMILY)); + admin.createTable(descriptor); + } + + @Override + protected void preConnectionClose() throws IOException { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + @Test + public void testCreateTable() throws IOException { + HTableDescriptor retrievedDescriptor = admin.getTableDescriptor(tableName); + Assert.assertEquals(descriptor.getTableName(), retrievedDescriptor.getTableName()); + } + + @Test + @Category(KnownGap.class) + public void testCreateTableFull() throws IOException { + HTableDescriptor retrievedDescriptor = admin.getTableDescriptor(tableName); + Assert.assertEquals(descriptor, retrievedDescriptor); + } + + @Test + public void testAddColumn() throws IOException { + HColumnDescriptor newColumn = new HColumnDescriptor("NEW_COLUMN"); + admin.addColumn(tableName, newColumn); + + // Make sure this call doesn't fail, but don't actually check the results yet, since Bigtable + // backend doesn't return column families yet. + admin.getTableDescriptor(tableName); + + HTableDescriptor expectedDescriptor = new HTableDescriptor(descriptor); + expectedDescriptor.addFamily(newColumn); + + admin.getTableDescriptor(tableName); + } + + @Test + @Category(KnownGap.class) + public void testAddAndCompareColumn() throws IOException { + HColumnDescriptor newColumn = new HColumnDescriptor("NEW_COLUMN"); + admin.addColumn(tableName, newColumn); + + HTableDescriptor expectedDescriptor = new HTableDescriptor(descriptor); + expectedDescriptor.addFamily(newColumn); + + HTableDescriptor retrievedDescriptor = admin.getTableDescriptor(tableName); + Assert.assertEquals(expectedDescriptor, retrievedDescriptor); + } + + + @Test + @Category(KnownGap.class) + public void testRemoveColumn() throws IOException { + admin.deleteColumn(tableName, DELETE_COLUMN_FAMILY); + HTableDescriptor retrievedDescriptor = admin.getTableDescriptor(tableName); + + HTableDescriptor expectedDescriptor = new HTableDescriptor(tableName); + expectedDescriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); + + Assert.assertEquals(expectedDescriptor, retrievedDescriptor); + } +} diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestCreateTable.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestCreateTable.java index a8acaa4c49..eb412c6898 100644 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestCreateTable.java +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestCreateTable.java @@ -13,7 +13,7 @@ */ package com.google.cloud.bigtable.hbase; -import static com.google.cloud.bigtable.hbase.IntegrationTests.*; +import static com.google.cloud.bigtable.hbase.IntegrationTests.COLUMN_FAMILY; import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -22,7 +22,6 @@ import org.apache.hadoop.hbase.client.Admin; import org.junit.Assert; import org.junit.Test; -import org.junit.experimental.categories.Category; import java.io.IOException; @@ -31,7 +30,6 @@ public class TestCreateTable extends AbstractTest { * Requirement 1.8 - Table names must match [_a-zA-Z0-9][-_.a-zA-Z0-9]* */ @Test - @Category(KnownGap.class) public void testTableNames() throws IOException { String[] goodNames = { "a", @@ -40,7 +38,10 @@ public void testTableNames() throws IOException { "_x", "a-._5x", "_a-._5x", - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890_-." + // TODO(sduskis): Join the last 2 strings once the Bigtable backend supports table names + // longer than 50 characters. + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghi", + "jklmnopqrstuvwxyz1234567890_-." }; String[] badNames = { "-x", diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestGet.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestGet.java index 0881e4f42f..f772409faa 100644 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestGet.java +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/TestGet.java @@ -462,7 +462,6 @@ public void testExists() throws IOException { * Requirement 3.16 - When submitting an array of Get operations, if one fails, they all fail. */ @Test - @Category(KnownGap.class) public void testOneBadApple() throws IOException { // Initialize data Table table = connection.getTable(TABLE_NAME); diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestBigtableRowAdapter.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestBigtableRowAdapter.java deleted file mode 100644 index 1f429589ac..0000000000 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestBigtableRowAdapter.java +++ /dev/null @@ -1,164 +0,0 @@ -package com.google.cloud.bigtable.hbase.adapters; - -import com.google.bigtable.v1.Cell; -import com.google.bigtable.v1.Column; -import com.google.bigtable.v1.Family; -import com.google.bigtable.v1.Row; -import com.google.cloud.bigtable.hbase.DataGenerationHelper; -import com.google.protobuf.ByteString; - -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Test for BigtableRowAdapter. - */ -@RunWith(JUnit4.class) -public class TestBigtableRowAdapter { - protected BigtableRowAdapter adapter = new BigtableRowAdapter(); - protected DataGenerationHelper dataHelper = new DataGenerationHelper(); - - @Test - public void testRowKeyNotPresentWhenNoCellsReturned() { - byte[] rowKey = dataHelper.randomData("rk1-"); - Row.Builder rowBuilder = Row.newBuilder(); - rowBuilder.setKey(ByteString.copyFrom(rowKey)); - - Result result = adapter.adaptResponse(rowBuilder.build()); - Assert.assertNull(result.getRow()); - } - - @Test - public void testNullIsAnEmptyResult() { - Row row = null; - Result result = adapter.adaptResponse(row); - Assert.assertNull(result.getRow()); - } - - @Test - public void testSingleCellResult() { - byte[] rowKey = dataHelper.randomData("rk1-"); - String columnFamily = "cf"; - byte[] qualfier = dataHelper.randomData("col1"); - byte[] value = dataHelper.randomData("val1"); - long bigtableTimestamp = 10000L; - - Row.Builder rowBuilder = Row.newBuilder(); - rowBuilder.setKey(ByteString.copyFrom(rowKey)); - - Family.Builder familyBuilder = rowBuilder.addFamiliesBuilder(); - familyBuilder.setName(columnFamily); - - Column.Builder columnBuilder = familyBuilder.addColumnsBuilder(); - columnBuilder.setQualifier(ByteString.copyFrom(qualfier)); - - Cell.Builder cellBuilder = columnBuilder.addCellsBuilder(); - cellBuilder.setValue(ByteString.copyFrom(value)); - cellBuilder.setTimestampMicros(bigtableTimestamp); - - Result result = adapter.adaptResponse(rowBuilder.build()); - Assert.assertEquals(1, result.size()); - - org.apache.hadoop.hbase.Cell cell = result.listCells().get(0); - Assert.assertArrayEquals(value, CellUtil.cloneValue(cell)); - Assert.assertArrayEquals(Bytes.toBytes(columnFamily), CellUtil.cloneFamily(cell)); - Assert.assertArrayEquals(qualfier, CellUtil.cloneQualifier(cell)); - Assert.assertArrayEquals(rowKey, CellUtil.cloneRow(cell)); - Assert.assertEquals(bigtableTimestamp / 1000L, cell.getTimestamp()); - } - - @Test - public void testMultipleCellsInSingleColumnResult() { - byte[] rowKey = dataHelper.randomData("rk1-"); - String columnFamily = "cf"; - byte[] qualfier = dataHelper.randomData("col1"); - byte[] value1 = dataHelper.randomData("val1"); - long bigtableTimestamp1 = 20000L; - byte[] value2 = dataHelper.randomData("val2"); - long bigtableTimestamp2 = 10000L; - - Row.Builder rowBuilder = Row.newBuilder(); - rowBuilder.setKey(ByteString.copyFrom(rowKey)); - - Family.Builder familyBuilder = rowBuilder.addFamiliesBuilder(); - familyBuilder.setName(columnFamily); - - Column.Builder columnBuilder = familyBuilder.addColumnsBuilder(); - columnBuilder.setQualifier(ByteString.copyFrom(qualfier)); - - Cell.Builder cellBuilder = columnBuilder.addCellsBuilder(); - cellBuilder.setValue(ByteString.copyFrom(value1)); - cellBuilder.setTimestampMicros(bigtableTimestamp1); - - Cell.Builder cellBuilder2 = columnBuilder.addCellsBuilder(); - cellBuilder2.setValue(ByteString.copyFrom(value2)); - cellBuilder2.setTimestampMicros(bigtableTimestamp2); - - Result result = adapter.adaptResponse(rowBuilder.build()); - Assert.assertEquals(2, result.size()); - - org.apache.hadoop.hbase.Cell cell1 = result.listCells().get(0); - Assert.assertArrayEquals(Bytes.toBytes(columnFamily), CellUtil.cloneFamily(cell1)); - Assert.assertArrayEquals(qualfier, CellUtil.cloneQualifier(cell1)); - Assert.assertArrayEquals(value1, CellUtil.cloneValue(cell1)); - Assert.assertEquals(bigtableTimestamp1 / 1000L, cell1.getTimestamp()); - - org.apache.hadoop.hbase.Cell cell2 = result.listCells().get(1); - Assert.assertArrayEquals(Bytes.toBytes(columnFamily), CellUtil.cloneFamily(cell2)); - Assert.assertArrayEquals(qualfier, CellUtil.cloneQualifier(cell2)); - Assert.assertArrayEquals(value2, CellUtil.cloneValue(cell2)); - Assert.assertEquals(bigtableTimestamp2 / 1000L, cell2.getTimestamp()); - } - - @Test - public void testMultipleCellsAreSortedByColumnName() { - byte[] rowKey = dataHelper.randomData("rk1-"); - String columnFamily = "cf"; - byte[] qualfierA = dataHelper.randomData("colA"); - byte[] qualfierB = dataHelper.randomData("colB"); - byte[] valueA = dataHelper.randomData("valA"); - long bigtableTimestampA = 10000L; - byte[] valueB = dataHelper.randomData("valB"); - long bigtableTimestampB = 10000L; - - Row.Builder rowBuilder = Row.newBuilder(); - rowBuilder.setKey(ByteString.copyFrom(rowKey)); - - Family.Builder familyBuilder = rowBuilder.addFamiliesBuilder(); - familyBuilder.setName(columnFamily); - - Column.Builder columnBuilderB = familyBuilder.addColumnsBuilder(); - columnBuilderB.setQualifier(ByteString.copyFrom(qualfierB)); - - Cell.Builder cellBuilderB = columnBuilderB.addCellsBuilder(); - cellBuilderB.setValue(ByteString.copyFrom(valueB)); - cellBuilderB.setTimestampMicros(bigtableTimestampB); - - Column.Builder columnBuilderA = familyBuilder.addColumnsBuilder(); - columnBuilderA.setQualifier(ByteString.copyFrom(qualfierA)); - - Cell.Builder cellBuilderA = columnBuilderA.addCellsBuilder(); - cellBuilderA.setValue(ByteString.copyFrom(valueA)); - cellBuilderA.setTimestampMicros(bigtableTimestampA); - - Result result = adapter.adaptResponse(rowBuilder.build()); - Assert.assertEquals(2, result.size()); - - org.apache.hadoop.hbase.Cell cellA = result.listCells().get(0); - Assert.assertArrayEquals(Bytes.toBytes(columnFamily), CellUtil.cloneFamily(cellA)); - Assert.assertArrayEquals(qualfierA, CellUtil.cloneQualifier(cellA)); - Assert.assertArrayEquals(valueA, CellUtil.cloneValue(cellA)); - Assert.assertEquals(bigtableTimestampA / 1000L, cellA.getTimestamp()); - - org.apache.hadoop.hbase.Cell cellB = result.listCells().get(1); - Assert.assertArrayEquals(Bytes.toBytes(columnFamily), CellUtil.cloneFamily(cellB)); - Assert.assertArrayEquals(qualfierB, CellUtil.cloneQualifier(cellB)); - Assert.assertArrayEquals(valueB, CellUtil.cloneValue(cellB)); - Assert.assertEquals(bigtableTimestampB / 1000L, cellB.getTimestamp()); - } -} diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestClusterMetadataSetter.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestClusterMetadataSetter.java new file mode 100644 index 0000000000..8343acbb4c --- /dev/null +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestClusterMetadataSetter.java @@ -0,0 +1,73 @@ +package com.google.cloud.bigtable.hbase.adapters; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import com.google.bigtable.admin.table.v1.CreateTableRequest; +import com.google.bigtable.admin.table.v1.ListTablesRequest; + +@RunWith(JUnit4.class) +public class TestClusterMetadataSetter { + + public static ClusterMetadataSetter clusterMetadataSetter = new ClusterMetadataSetter( + "some-project", "some-zone", "some-cluster"); + private String clusterId = "projects/some-project/zones/some-zone/clusters/some-cluster"; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testFormat() { + Assert.assertEquals(clusterId, clusterMetadataSetter.getFormattedV1ClusterName()); + } + + @Test + public void testListTablesRequest() { + ListTablesRequest.Builder builder = ListTablesRequest.newBuilder(); + clusterMetadataSetter.setMetadata(builder); + Assert.assertEquals(clusterId, builder.getName()); + } + + @Test + public void testCreateTablesRequest() { + CreateTableRequest.Builder builder = CreateTableRequest.newBuilder(); + clusterMetadataSetter.setMetadata(builder); + Assert.assertEquals(clusterId, builder.getName()); + } + + @Test + public void testGoodTableQualifier() { + clusterMetadataSetter.toHBaseTableName(clusterId + "/" + TableMetadataSetter.TABLE_SEPARATOR + + "/foo"); + } + + @Test + public void testNullQualifier() { + expectedException.expect(NullPointerException.class); + clusterMetadataSetter.toHBaseTableName(null); + } + + @Test + public void testBadQualifier() { + expectedException.expect(IllegalStateException.class); + clusterMetadataSetter.toHBaseTableName(clusterId.replace("some-cluster", "another-cluster") + + "/" + TableMetadataSetter.TABLE_SEPARATOR + "/foo"); + } + + @Test + public void testBlankTableName() { + expectedException.expect(IllegalStateException.class); + clusterMetadataSetter.toHBaseTableName(clusterId + "/" + TableMetadataSetter.TABLE_SEPARATOR + + "/"); + } + + @Test + public void testNoTableName() { + expectedException.expect(IllegalStateException.class); + clusterMetadataSetter.toHBaseTableName(clusterId + "/" + TableMetadataSetter.TABLE_SEPARATOR); + } +} diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestColumnDescriptorAdapter.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestColumnDescriptorAdapter.java index 998e7dfe2d..113a8432e5 100644 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestColumnDescriptorAdapter.java +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestColumnDescriptorAdapter.java @@ -1,7 +1,6 @@ package com.google.cloud.bigtable.hbase.adapters; -import com.google.bigtable.anviltop.AnviltopData; -import com.google.cloud.bigtable.hbase.adapters.ColumnDescriptorAdapter; +import com.google.bigtable.admin.table.v1.ColumnFamily; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.io.compress.Compression; @@ -9,6 +8,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -24,8 +24,8 @@ @RunWith(JUnit4.class) public class TestColumnDescriptorAdapter { - ColumnDescriptorAdapter adapter; - HColumnDescriptor descriptor; + private ColumnDescriptorAdapter adapter; + private HColumnDescriptor descriptor; @Before public void setup() { @@ -81,7 +81,7 @@ public void ignoredOptionsAreIgnored() { descriptor.setScope(1); // REPLICATION_SCOPE descriptor.setInMemory(true); - AnviltopData.ColumnFamily.Builder result = adapter.adapt(descriptor) + ColumnFamily.Builder result = adapter.adapt(descriptor) .clearName() .clearGcExpression(); @@ -91,42 +91,93 @@ public void ignoredOptionsAreIgnored() { } @Test + @Ignore("The name of the column family is in a higher level object like Table or the create column request") public void columnFamilyNameIsPreserved() { - Assert.assertEquals(descriptor.getNameAsString(), adapter.adapt(descriptor).getName()); + String adapted = adapter.adapt(descriptor).getName(); + Assert.assertTrue(adapted.endsWith(descriptor.getNameAsString())); } @Test public void ttlIsPreservedInGcExpression() { // TTL of 1 day (in seconds): descriptor.setTimeToLive(86400); - AnviltopData.ColumnFamilyOrBuilder result = adapter.adapt(descriptor); - + ColumnFamily.Builder result = adapter.adapt(descriptor); Assert.assertEquals("(age() > 86400000000) || (version() > 1)", result.getGcExpression()); } + @Test + public void ttlIsPreservedInColumnFamily() { + // TTL of 1 day (in microseconds): + String expression = "(age() > 86400000000) || (version() > 1)"; + HColumnDescriptor descriptor = adapter.adapt("family", asColumnFamily(expression)); + Assert.assertEquals(1, descriptor.getMaxVersions()); + Assert.assertEquals(86400, descriptor.getTimeToLive()); + } + @Test public void maxVersionsIsPreservedInGcExpression() { descriptor.setMaxVersions(10); - AnviltopData.ColumnFamilyOrBuilder result = adapter.adapt(descriptor); + ColumnFamily.Builder result = adapter.adapt(descriptor); Assert.assertEquals("(version() > 10)", result.getGcExpression()); } @Test - public void minAndMaxMayBeSpecifiedTogetherWIthTtl() { + public void maxVersionsIsPreservedInColumnFamily() { + String expression = "(version() > 10)"; + HColumnDescriptor descriptor = adapter.adapt("family", asColumnFamily(expression)); + Assert.assertEquals(10, descriptor.getMaxVersions()); + } + + @Test + public void minMaxTtlInDescriptor() { descriptor.setMaxVersions(20); descriptor.setMinVersions(10); descriptor.setTimeToLive(86400); // 1 day in seconds - AnviltopData.ColumnFamilyOrBuilder result = adapter.adapt(descriptor); + ColumnFamily.Builder result = adapter.adapt(descriptor); Assert.assertEquals( "(age() > 86400000000 && version() > 10) || (version() > 20)", result.getGcExpression()); } @Test - public void minVersionsMustBeLessThanMaxVersions() { + public void minMaxTtlInColumnFamily() { + String expression = "(age() > 86400000000 && version() > 10) || (version() > 20)"; + HColumnDescriptor descriptor = adapter.adapt("family", asColumnFamily(expression)); + Assert.assertEquals(20, descriptor.getMaxVersions()); + Assert.assertEquals(10, descriptor.getMinVersions()); + Assert.assertEquals(86400, descriptor.getTimeToLive()); + } + + public static ColumnFamily asColumnFamily(String expression) { + return ColumnFamily.newBuilder().setGcExpression(expression).build(); + } + + @Test + public void minVersionsMustBeLessThanMaxversion() { descriptor.setMaxVersions(10); descriptor.setMinVersions(20); expectedException.expect(IllegalStateException.class); adapter.adapt(descriptor); } + + @Test + public void minVersionsMustBeLessThanMaxversionInExpression() { + String expression = "(age() > 86400000000 && version() > 20) || (version() > 10)"; + expectedException.expect(IllegalStateException.class); + adapter.adapt("family", asColumnFamily(expression)); + } + + @Test + public void badExpressionLessThan(){ + String expression = "(version() < 1)"; + expectedException.expect(IllegalStateException.class); + adapter.adapt("family", asColumnFamily(expression)); + } + + @Test + public void badExpressionBlank(){ + String expression = ""; + expectedException.expect(IllegalStateException.class); + adapter.adapt("family", asColumnFamily(expression)); + } } diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestDeleteAdapter.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestDeleteAdapter.java index a025acc8a8..7d7a40b38a 100644 --- a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestDeleteAdapter.java +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestDeleteAdapter.java @@ -13,11 +13,7 @@ */ package com.google.cloud.bigtable.hbase.adapters; -import com.google.bigtable.v1.MutateRowRequest; -import com.google.bigtable.v1.Mutation; -import com.google.bigtable.v1.Mutation.MutationCase; -import com.google.bigtable.v1.TimestampRange; -import com.google.cloud.bigtable.hbase.DataGenerationHelper; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.util.Bytes; @@ -28,7 +24,11 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import java.util.concurrent.TimeUnit; +import com.google.bigtable.v1.MutateRowRequest; +import com.google.bigtable.v1.Mutation; +import com.google.bigtable.v1.Mutation.MutationCase; +import com.google.bigtable.v1.TimestampRange; +import com.google.cloud.bigtable.hbase.DataGenerationHelper; @RunWith(JUnit4.class) diff --git a/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestTableMetadataSetter.java b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestTableMetadataSetter.java new file mode 100644 index 0000000000..aa7689399b --- /dev/null +++ b/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/adapters/TestTableMetadataSetter.java @@ -0,0 +1,27 @@ +package com.google.cloud.bigtable.hbase.adapters; + +import org.apache.hadoop.hbase.TableName; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class TestTableMetadataSetter { + + ClusterMetadataSetter clusterMetadataSetter = + new ClusterMetadataSetter("some-project", "some-zone", "some-cluster"); + + String tableName = "some-table"; + TableMetadataSetter tableMetadataSetter = + new TableMetadataSetter(TableName.valueOf(tableName), "some-project", "some-zone", + "some-cluster"); + + @Test + public void testGoodHBaseName() { + String hbaseName = + clusterMetadataSetter.toHBaseTableName(tableMetadataSetter.getFormattedV1TableName()); + + Assert.assertEquals(tableName, hbaseName); + } +}