From ab0513f912be18b7ccd656d657884d05f66a4378 Mon Sep 17 00:00:00 2001 From: coufon Date: Sat, 22 Apr 2023 21:27:37 +0000 Subject: [PATCH 01/22] Add Iceberg Catalog for Google BigLake Metastore --- .gitignore | 3 + build.gradle | 26 + .../iceberg/gcp/biglake/BigLakeCatalog.java | 405 ++++++++++++++ .../iceberg/gcp/biglake/BigLakeClient.java | 137 +++++ .../gcp/biglake/BigLakeClientImpl.java | 288 ++++++++++ .../gcp/biglake/BigLakeTableOperations.java | 229 ++++++++ .../gcp/biglake/BigLakeCatalogTest.java | 494 ++++++++++++++++++ .../biglake/BigLakeTableOperationsTest.java | 221 ++++++++ .../iceberg/gcp/biglake/BigLakeTestUtils.java | 82 +++ spark/v3.1/build.gradle | 1 + spark/v3.2/build.gradle | 1 + spark/v3.3/build.gradle | 1 + spark/v3.4/build.gradle | 1 + versions.props | 1 + 14 files changed, 1890 insertions(+) create mode 100644 gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java create mode 100644 gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java create mode 100644 gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java create mode 100644 gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java create mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java create mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java create mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java diff --git a/.gitignore b/.gitignore index 13e95b24648b..5d1b337832f3 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ derby.log python/.mypy_cache/ python/htmlcov python/coverage.xml + +# GCP test files +gcp/db_folder/ diff --git a/build.gradle b/build.gradle index cef0cfbe0ad1..4b227a5eab75 100644 --- a/build.gradle +++ b/build.gradle @@ -594,6 +594,10 @@ project(':iceberg-delta-lake') { } project(':iceberg-gcp') { + test { + useJUnitPlatform() + } + dependencies { implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') api project(':iceberg-api') @@ -602,6 +606,28 @@ project(':iceberg-gcp') { implementation platform('com.google.cloud:libraries-bom') implementation 'com.google.cloud:google-cloud-storage' + implementation 'com.google.cloud:google-cloud-biglake' + compileOnly('org.apache.hadoop:hadoop-common') { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'javax.servlet', module: 'servlet-api' + exclude group: 'com.google.code.gson', module: 'gson' + } + compileOnly("org.apache.hive:hive-metastore") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.pentaho' // missing dependency + exclude group: 'org.apache.hbase' + exclude group: 'org.apache.logging.log4j' + exclude group: 'co.cask.tephra' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all' + exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet' + exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle' + exclude group: 'com.tdunning', module: 'json' + exclude group: 'javax.transaction', module: 'transaction-api' + exclude group: 'com.zaxxer', module: 'HikariCP' + } testImplementation 'com.google.cloud:google-cloud-nio' diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java new file mode 100644 index 000000000000..0a57b906e4e2 --- /dev/null +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.BaseMetastoreCatalog; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.hadoop.Util; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Strings; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Streams; +import org.apache.iceberg.util.LocationUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ +public final class BigLakeCatalog extends BaseMetastoreCatalog + implements SupportsNamespaces, Configurable { + + // User provided properties. + // The endpoint of BigLake API. Optional, default to DEFAULT_BIGLAKE_SERVICE_ENDPOINT. + public static final String PROPERTIES_KEY_BIGLAKE_ENDPOINT = "blms_endpoint"; + // The GCP project ID. Required. + public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp_project"; + // The GCP location (https://cloud.google.com/bigquery/docs/locations). Optional, default to + // DEFAULT_GCP_LOCATION. + public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp_location"; + // The BLMS catalog ID. It is the container resource of databases and tables. + // It links a BLMS catalog with this Iceberg catalog. + public static final String PROPERTIES_KEY_BLMS_CATALOG = "blms_catalog"; + + public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir"; + + public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; + public static final String DEFAULT_GCP_LOCATION = "us"; + + private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); + + // The name of this Iceberg catalog plugin: spark.sql.catalog.. + private String catalogPulginName; + private Map catalogProperties; + private FileSystem fs; + private FileIO fileIO; + private Configuration conf; + private String projectId; + private String location; + // BLMS catalog ID and fully qualified name. + private String catalogId; + private CatalogName catalogName; + private BigLakeClient client; + + // Must have a no-arg constructor to be dynamically loaded + // initialize(String name, Map properties) will be called to complete + // initialization + public BigLakeCatalog() {} + + @Override + public void initialize(String inputName, Map properties) { + if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) { + throw new ValidationException("GCP project must be specified"); + } + String propProjectId = properties.get(PROPERTIES_KEY_GCP_PROJECT); + String propLocation = + properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION); + BigLakeClient newClient; + try { + newClient = + new BigLakeClientImpl( + properties.getOrDefault( + PROPERTIES_KEY_BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), + propProjectId, + propLocation); + } catch (IOException e) { + throw new ServiceFailureException(e, "Creating BigLake client failed"); + } + initialize(inputName, properties, propProjectId, propLocation, newClient); + } + + @VisibleForTesting + void initialize( + String inputName, + Map properties, + String propProjectId, + String propLocation, + BigLakeClient bigLakeClient) { + this.catalogPulginName = inputName; + this.catalogProperties = ImmutableMap.copyOf(properties); + this.projectId = propProjectId; + this.location = propLocation; + Preconditions.checkNotNull(bigLakeClient, "BigLake client must not be null"); + this.client = bigLakeClient; + + if (this.conf == null) { + LOG.warn("No Hadoop Configuration was set, using the default environment Configuration"); + this.conf = new Configuration(); + } + + // Users can specify the BigLake catalog ID, otherwise catalog plugin will be used. + this.catalogId = properties.getOrDefault(PROPERTIES_KEY_BLMS_CATALOG, inputName); + this.catalogName = CatalogName.of(projectId, location, catalogId); + LOG.info("Use BigLake catalog: {}", catalogName.toString()); + + if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) { + this.conf.set( + HIVE_METASTORE_WAREHOUSE_DIR, + LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION))); + } + + this.fs = + Util.getFs( + new Path( + LocationUtil.stripTrailingSlash( + properties.get(CatalogProperties.WAREHOUSE_LOCATION))), + conf); + + String fileIOImpl = + properties.getOrDefault( + CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO"); + this.fileIO = CatalogUtil.loadFileIO(fileIOImpl, properties, conf); + } + + @Override + protected TableOperations newTableOps(TableIdentifier identifier) { + return new BigLakeTableOperations( + conf, + client, + fileIO, + getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); + } + + @Override + protected String defaultWarehouseLocation(TableIdentifier identifier) { + String locationUri = getDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); + return String.format( + "%s/%s", + Strings.isNullOrEmpty(locationUri) + ? getDatabaseLocation(getDatabaseId(identifier.namespace())) + : locationUri, + identifier.name()); + } + + @Override + public List listTables(Namespace namespace) { + // When deleting a BLMS catalog via `DROP NAMESPACE `, this method is called for + // verifying catalog emptiness. `namespace` is empty in this case, we list databases in + // this catalog instead. + if (namespace.levels().length == 0) { + return Iterables.isEmpty(client.listDatabases(catalogName)) + ? ImmutableList.of() + : ImmutableList.of(TableIdentifier.of("placeholder")); + } + return Streams.stream(client.listTables(getDatabaseName(namespace))) + .map(BigLakeCatalog::getTableIdentifier) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public boolean dropTable(TableIdentifier identifier, boolean purge) { + TableOperations ops = newTableOps(identifier); + // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. + TableMetadata lastMetadata = ops.current(); + client.deleteTable( + getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); + if (purge && lastMetadata != null) { + CatalogUtil.dropTableData(ops.io(), lastMetadata); + } + return true; + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) { + String fromDbId = getDatabaseId(from.namespace()); + String toDbId = getDatabaseId(to.namespace()); + + if (!fromDbId.equals(toDbId)) { + throw new ValidationException("New table name must be in the same database"); + } + + client.renameTable(getTableName(fromDbId, from.name()), getTableName(toDbId, to.name())); + } + + @Override + public void createNamespace(Namespace namespace, Map metadata) { + if (namespace.levels().length == 0) { + // Used by `CREATE NAMESPACE `. Create a BLMS catalog linked with Iceberg catalog. + client.createCatalog(catalogName, Catalog.getDefaultInstance()); + LOG.info("Created BigLake catalog: {}", catalogName.toString()); + } else if (namespace.levels().length == 1) { + // Create a database. + String dbId = namespace.level(0); + Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); + builder + .getHiveOptionsBuilder() + .putAllParameters(metadata) + .setLocationUri(getDatabaseLocation(dbId)); + + Database db = + client.createDatabase( + DatabaseName.of(projectId, location, catalogId, dbId), builder.build()); + // Creates the data folder for the database. + try { + fs.mkdirs(new Path(db.getHiveOptions().getLocationUri())); + } catch (IOException e) { + throw new UncheckedIOException(String.format("Create namespace failed: %s", namespace), e); + } + } else { + throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + } + } + + @Override + public List listNamespaces(Namespace namespace) { + if (namespace.levels().length != 0) { + // BLMS does not support namespaces under database or tables, returns empty. + // It is called when dropping a namespace to make sure it's empty (listTables is called as + // well), returns empty to unblock deletion. + return ImmutableList.of(); + } + return Streams.stream(client.listDatabases(catalogName)) + .map(BigLakeCatalog::getNamespace) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public boolean dropNamespace(Namespace namespace) { + if (namespace.levels().length == 0) { + // Used by `DROP NAMESPACE `. Deletes the BLMS catalog linked by Iceberg catalog. + client.deleteCatalog(catalogName); + LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); + } else if (namespace.levels().length == 1) { + client.deleteDatabase(getDatabaseName(namespace)); + // We don't delete the data file folder for safety. It aligns with HMS's default behavior. + // We can support database or catalog level config controlling file deletion in future. + } else { + throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + } + return true; + } + + @Override + public boolean setProperties(Namespace namespace, Map properties) { + Database.Builder builder; + try { + builder = getDatabase(namespace).toBuilder(); + } catch (IllegalArgumentException e) { + LOG.warn( + "setProperties is only supported for tables and databases, namespace {} is not supported", + namespace.levels().length == 0 ? "empty" : namespace.toString(), + e); + return false; + } + HiveDatabaseOptions.Builder optionsBuilder = builder.getHiveOptionsBuilder(); + properties.forEach(optionsBuilder::putParameters); + client.updateDatabaseParameters(getDatabaseName(namespace), optionsBuilder.getParametersMap()); + return true; + } + + @Override + public boolean removeProperties(Namespace namespace, Set properties) { + Database.Builder builder; + try { + builder = getDatabase(namespace).toBuilder(); + } catch (IllegalArgumentException e) { + LOG.warn( + "removeProperties is only supported for tables and databases, namespace {} is not" + + " supported", + namespace.levels().length == 0 ? "empty" : namespace.toString(), + e); + return false; + } + HiveDatabaseOptions.Builder optionsBuilder = builder.getHiveOptionsBuilder(); + properties.forEach(optionsBuilder::removeParameters); + client.updateDatabaseParameters(getDatabaseName(namespace), optionsBuilder.getParametersMap()); + return true; + } + + @Override + public Map loadNamespaceMetadata(Namespace namespace) { + if (namespace.levels().length == 0) { + // Calls getCatalog to check existence. BLMS catalog has no metadata today. + client.getCatalog(catalogName); + return new HashMap(); + } else if (namespace.levels().length == 1) { + return getMetadata(getDatabase(namespace)); + } else { + throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + } + } + + @Override + public String name() { + return catalogPulginName; + } + + @Override + protected Map properties() { + return catalogProperties == null ? ImmutableMap.of() : catalogProperties; + } + + @Override + public void setConf(Configuration conf) { + this.conf = new Configuration(conf); + } + + @Override + public Configuration getConf() { + return this.conf; + } + + private String getDatabaseLocation(String dbId) { + String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR); + Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set"); + return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); + } + + private static TableIdentifier getTableIdentifier(Table table) { + TableName tableName = TableName.parse(table.getName()); + return TableIdentifier.of(Namespace.of(tableName.getDatabase()), tableName.getTable()); + } + + private static Namespace getNamespace(Database db) { + return Namespace.of(DatabaseName.parse(db.getName()).getDatabase()); + } + + private TableName getTableName(String dbId, String tableId) { + return TableName.of(projectId, location, catalogId, dbId, tableId); + } + + private String getDatabaseId(Namespace namespace) { + Preconditions.checkArgument( + namespace.levels().length == 1, + "BigLake database namespace must use format ., invalid namespace: %s", + namespace); + return namespace.level(0); + } + + private DatabaseName getDatabaseName(Namespace namespace) { + return DatabaseName.of(projectId, location, catalogId, getDatabaseId(namespace)); + } + + private Database getDatabase(Namespace namespace) { + return client.getDatabase(getDatabaseName(namespace)); + } + + private static Map getMetadata(Database db) { + HiveDatabaseOptions options = db.getHiveOptions(); + Map result = Maps.newHashMap(); + result.putAll(options.getParameters()); + result.put("location", options.getLocationUri()); + return result; + } + + private static String invalidNamespaceMessage(Namespace namespace) { + return String.format( + "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + + " namespace: %s", + namespace); + } +} diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java new file mode 100644 index 000000000000..4373f4d2242a --- /dev/null +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.util.Map; + +/** A client interface of Google BigLake service. */ +public interface BigLakeClient { + + /** + * Creates and returns a new catalog. + * + * @param name full catalog name + * @param catalog body of catalog to create + */ + Catalog createCatalog(CatalogName name, Catalog catalog); + + /** + * Returns a catalog. + * + * @param name full catalog name + */ + Catalog getCatalog(CatalogName name); + + /** + * Deletes a catalog. + * + * @param name full catalog name + */ + void deleteCatalog(CatalogName name); + + /** + * Creates and returns a new database. + * + * @param name full database name + * @param db body of database to create + */ + Database createDatabase(DatabaseName name, Database db); + + /** + * Returns a database. + * + * @param name full database name + */ + Database getDatabase(DatabaseName name); + + /** + * Updates the parameters of a Hive database and returns the updated database. + * + * @param name full database name + * @param parameters Hive options parameters to fully update + */ + Database updateDatabaseParameters(DatabaseName name, Map parameters); + + /** + * Returns all databases in a catalog. + * + * @param name full catalog name + */ + Iterable listDatabases(CatalogName name); + + /** + * Deletes a database. + * + * @param name full database name + */ + void deleteDatabase(DatabaseName name); + + /** + * Creates and returns a new table. + * + * @param name full database name + * @param table body of table to create + */ + Table createTable(TableName name, Table table); + + /** + * Returns a table. + * + * @param name full table name + */ + Table getTable(TableName name); + + /** + * Updates the parameters of a Hive table and returns the updated table. + * + * @param name full table name + * @param parameters Hive options parameters to fully update + * @param etag representation of table fields for concurrent update detection, see + * https://www.rfc-editor.org/rfc/rfc7232#section-2.3 + */ + Table updateTableParameters(TableName name, Map parameters, String etag); + + /** + * Renames a table. + * + * @param name full table name + * @param newName new full table name + */ + Table renameTable(TableName name, TableName newName); + + /** + * Deletes a table. + * + * @param name full table name + */ + Table deleteTable(TableName name); + + /** + * Returns all tables in a database. + * + * @param name full database name + */ + Iterable listTables(DatabaseName name); +} diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java new file mode 100644 index 000000000000..12cb9b257555 --- /dev/null +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.api.gax.rpc.PermissionDeniedException; +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.CreateCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.CreateDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.CreateTableRequest; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.DeleteCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.DeleteDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.DeleteTableRequest; +import com.google.cloud.bigquery.biglake.v1.GetCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.GetDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.GetTableRequest; +import com.google.cloud.bigquery.biglake.v1.ListDatabasesRequest; +import com.google.cloud.bigquery.biglake.v1.ListTablesRequest; +import com.google.cloud.bigquery.biglake.v1.LocationName; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceClient; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import com.google.cloud.bigquery.biglake.v1.RenameTableRequest; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import com.google.cloud.bigquery.biglake.v1.UpdateDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.UpdateTableRequest; +import com.google.protobuf.Empty; +import com.google.protobuf.FieldMask; +import java.io.IOException; +import java.util.Map; +import java.util.function.Supplier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotAuthorizedException; + +/** A client implementation of Google BigLake service. */ +public final class BigLakeClientImpl implements BigLakeClient { + + private final String projectId; + private final String location; + private final MetastoreServiceClient stub; + + /** + * Constructs a client of Google BigLake Service. + * + * @param biglakeEndpoint BigLake service gRPC endpoint, e.g., "biglake.googleapis.com:443" + * @param projectId GCP project ID + * @param location GCP region supported by BigLake, e.g., "us" + */ + public BigLakeClientImpl(String biglakeEndpoint, String projectId, String location) + throws IOException { + this.projectId = projectId; + this.location = location; + this.stub = + MetastoreServiceClient.create( + MetastoreServiceSettings.newBuilder().setEndpoint(biglakeEndpoint).build()); + } + + @Override + public Catalog createCatalog(CatalogName name, Catalog catalog) { + return convertException( + () -> + stub.createCatalog( + CreateCatalogRequest.newBuilder() + .setParent(LocationName.of(name.getProject(), name.getLocation()).toString()) + .setCatalogId(name.getCatalog()) + .setCatalog(catalog) + .build())); + } + + @Override + public Catalog getCatalog(CatalogName name) { + return convertException( + () -> { + try { + return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Catalog %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public void deleteCatalog(CatalogName name) { + convertException( + () -> { + try { + stub.deleteCatalog(DeleteCatalogRequest.newBuilder().setName(name.toString()).build()); + return Empty.getDefaultInstance(); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Catalog %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Database createDatabase(DatabaseName name, Database db) { + return convertException( + () -> + stub.createDatabase( + CreateDatabaseRequest.newBuilder() + .setParent( + CatalogName.of(name.getProject(), name.getLocation(), name.getCatalog()) + .toString()) + .setDatabaseId(name.getDatabase()) + .setDatabase(db) + .build())); + } + + @Override + public Database getDatabase(DatabaseName name) { + return convertException( + () -> { + try { + return stub.getDatabase( + GetDatabaseRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Database %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Database updateDatabaseParameters(DatabaseName name, Map parameters) { + Database.Builder builder = Database.newBuilder().setName(name.toString()); + builder.getHiveOptionsBuilder().putAllParameters(parameters); + return convertException( + () -> { + try { + return stub.updateDatabase( + UpdateDatabaseRequest.newBuilder() + .setDatabase(builder) + .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Database %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Iterable listDatabases(CatalogName name) { + return convertException( + () -> + stub.listDatabases(ListDatabasesRequest.newBuilder().setParent(name.toString()).build()) + .iterateAll()); + } + + @Override + public void deleteDatabase(DatabaseName name) { + convertException( + () -> { + try { + stub.deleteDatabase( + DeleteDatabaseRequest.newBuilder().setName(name.toString()).build()); + return Empty.getDefaultInstance(); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Database %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Table createTable(TableName name, Table table) { + return convertException( + () -> + stub.createTable( + CreateTableRequest.newBuilder() + .setParent(getDatabase(name).toString()) + .setTableId(name.getTable()) + .setTable(table) + .build())); + } + + @Override + public Table getTable(TableName name) { + return convertException( + () -> { + try { + return stub.getTable(GetTableRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Table updateTableParameters(TableName name, Map parameters, String etag) { + Table.Builder builder = Table.newBuilder().setName(name.toString()).setEtag(etag); + builder.getHiveOptionsBuilder().putAllParameters(parameters); + return convertException( + () -> { + try { + return stub.updateTable( + UpdateTableRequest.newBuilder() + .setTable(builder) + .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Table renameTable(TableName name, TableName newName) { + return convertException( + () -> { + try { + return stub.renameTable( + RenameTableRequest.newBuilder() + .setName(name.toString()) + .setNewName(newName.toString()) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Table deleteTable(TableName name) { + return convertException( + () -> { + try { + return stub.deleteTable( + DeleteTableRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table %s not found or permission denied", name.toString()); + } + }); + } + + @Override + public Iterable
listTables(DatabaseName name) { + return convertException( + () -> + stub.listTables(ListTablesRequest.newBuilder().setParent(name.toString()).build()) + .iterateAll()); + } + + // Converts BigLake API errors to Iceberg errors. + private T convertException(Supplier result) { + try { + return result.get(); + } catch (PermissionDeniedException e) { + throw new NotAuthorizedException(e, "Not authorized to call BigLake API"); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "BigLake resource already exists"); + } + } + + private static DatabaseName getDatabase(TableName tableName) { + return DatabaseName.of( + tableName.getProject(), + tableName.getLocation(), + tableName.getCatalog(), + tableName.getDatabase()); + } +} diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java new file mode 100644 index 000000000000..ed22724470b1 --- /dev/null +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.api.gax.rpc.AbortedException; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions.SerDeInfo; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions.StorageDescriptor; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Handles BigLake table operations. */ +public final class BigLakeTableOperations extends BaseMetastoreTableOperations { + + private static final Logger LOG = LoggerFactory.getLogger(BigLakeTableOperations.class); + + private final Configuration conf; + private final BigLakeClient client; + private final FileIO fileIO; + private final TableName tableName; + + BigLakeTableOperations( + Configuration conf, BigLakeClient client, FileIO fileIO, TableName tableName) { + this.conf = conf; + this.client = client; + this.fileIO = fileIO; + this.tableName = tableName; + } + + // The doRefresh method should provide implementation on how to get the metadata location + @Override + public void doRefresh() { + // Must default to null. + String metadataLocation = null; + try { + HiveTableOptions hiveOptions = client.getTable(tableName).getHiveOptions(); + if (!hiveOptions.containsParameters(METADATA_LOCATION_PROP)) { + throw new ValidationException( + "Table %s is not a valid Iceberg table, metadata location not found", tableName()); + } + metadataLocation = hiveOptions.getParametersOrThrow(METADATA_LOCATION_PROP); + } catch (NoSuchTableException e) { + if (currentMetadataLocation() != null) { + // Re-throws the exception because the table must exist in this case. + throw e; + } + } + refreshFromMetadataLocation(metadataLocation); + } + + // The doCommit method should provide implementation on how to update with metadata location + // atomically + @Override + public void doCommit(TableMetadata base, TableMetadata metadata) { + boolean isNewTable = base == null; + String newMetadataLocation = writeNewMetadataIfRequired(isNewTable, metadata); + + CommitStatus commitStatus = CommitStatus.FAILURE; + try { + if (isNewTable) { + createTable(newMetadataLocation, metadata); + } else { + updateTable(base.metadataFileLocation(), newMetadataLocation, metadata); + } + commitStatus = CommitStatus.SUCCESS; + } catch (CommitFailedException | CommitStateUnknownException e) { + throw e; + } catch (Throwable e) { + commitStatus = checkCommitStatus(newMetadataLocation, metadata); + if (commitStatus == CommitStatus.FAILURE) { + throw new CommitFailedException(e, "Failed to commit"); + } + if (commitStatus == CommitStatus.UNKNOWN) { + throw new CommitStateUnknownException(e); + } + } finally { + try { + if (commitStatus == CommitStatus.FAILURE) { + LOG.warn("Failed to commit updates to table {}", tableName()); + io().deleteFile(newMetadataLocation); + } + } catch (RuntimeException e) { + LOG.error( + "Failed to cleanup metadata file at {} for table {}", + newMetadataLocation, + tableName(), + e); + } + } + } + + @Override + public String tableName() { + return String.format( + "%s.%s.%s", tableName.getCatalog(), tableName.getDatabase(), tableName.getTable()); + } + + @Override + public FileIO io() { + return fileIO; + } + + private void createTable(String newMetadataLocation, TableMetadata metadata) { + LOG.debug("Creating a new Iceberg table: {}", tableName()); + client.createTable(tableName, makeNewTable(metadata, newMetadataLocation)); + } + + /** Update table properties with concurrent update detection using etag. */ + private void updateTable( + String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) { + Table table = client.getTable(tableName); + String etag = table.getEtag(); + if (etag.isEmpty()) { + throw new ValidationException( + "Etag of legacy table %s is empty, manually update the table by BigLake API or" + + " recreate and retry", + tableName()); + } + HiveTableOptions options = table.getHiveOptions(); + + // If `metadataLocationFromMetastore` is different from metadata location of base, it means + // someone has updated metadata location in metastore, which is a conflict update. + String metadataLocationFromMetastore = + options.getParametersOrDefault(METADATA_LOCATION_PROP, ""); + if (!metadataLocationFromMetastore.isEmpty() + && !metadataLocationFromMetastore.equals(oldMetadataLocation)) { + throw new CommitFailedException( + "Base metadata location '%s' is not same as the current table metadata location '%s' for" + + " %s.%s", + oldMetadataLocation, + metadataLocationFromMetastore, + tableName.getDatabase(), + tableName.getTable()); + } + + try { + client.updateTableParameters( + tableName, buildTableParameters(newMetadataLocation, metadata), etag); + } catch (AbortedException e) { + if (e.getMessage().toLowerCase().contains("etag mismatch")) { + throw new CommitFailedException( + "Updating table failed due to conflict updates (etag mismatch)"); + } + } + } + + private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) { + Table.Builder builder = Table.newBuilder().setType(Table.Type.HIVE); + builder + .getHiveOptionsBuilder() + .setTableType("EXTERNAL_TABLE") + .setStorageDescriptor( + StorageDescriptor.newBuilder() + .setLocationUri(metadata.location()) + .setInputFormat("org.apache.hadoop.mapred.FileInputFormat") + .setOutputFormat("org.apache.hadoop.mapred.FileOutputFormat") + .setSerdeInfo( + SerDeInfo.newBuilder() + .setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))) + .putAllParameters(buildTableParameters(metadataFileLocation, metadata)); + return builder.build(); + } + + // Follow Iceberg's HiveTableOperations to populate more table parameters for HMS compatibility. + private Map buildTableParameters( + String metadataFileLocation, TableMetadata metadata) { + Map parameters = Maps.newHashMap(); + parameters.putAll(metadata.properties()); + if (metadata.uuid() != null) { + parameters.put(TableProperties.UUID, metadata.uuid()); + } + if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) { + parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation()); + } + parameters.put(METADATA_LOCATION_PROP, metadataFileLocation); + // Follow HMS to use the EXTERNAL type. + parameters.put("EXTERNAL", "TRUE"); + parameters.put("table_type", "ICEBERG"); + + // Hive style basic statistics. + if (metadata.currentSnapshot() != null) { + Map summary = metadata.currentSnapshot().summary(); + if (summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP) != null) { + parameters.put( + StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP)); + } + if (summary.get(SnapshotSummary.TOTAL_RECORDS_PROP) != null) { + parameters.put(StatsSetupConst.ROW_COUNT, summary.get(SnapshotSummary.TOTAL_RECORDS_PROP)); + } + if (summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP) != null) { + parameters.put( + StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP)); + } + } + // TODO: to expose more Iceberg metadata if needed, e.g., statistic, schema, partition spec. + return parameters; + } +} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java new file mode 100644 index 000000000000..497519d1600a --- /dev/null +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -0,0 +1,494 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.File; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigLakeCatalogTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + + private static final String GCP_PROJECT = "my-project"; + private static final String GCP_REGION = "us"; + private static final String CATALOG_ID = "biglake"; + + @Mock private BigLakeClient bigLakeClient; + + private BigLakeCatalog bigLakeCatalog; + private String warehouseLocation; + + @Before + public void before() throws Exception { + this.bigLakeCatalog = new BigLakeCatalog(); + this.warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + + bigLakeCatalog.initialize( + CATALOG_ID, + /* properties= */ ImmutableMap.of( + BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, + GCP_PROJECT, + CatalogProperties.WAREHOUSE_LOCATION, + warehouseLocation), + GCP_PROJECT, + GCP_REGION, + bigLakeClient); + } + + @Test + public void testDefaultWarehouseWithDatabaseLocation_asExpected() { + when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) + .build()); + + assertEquals( + "db_folder/table", + bigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + } + + @Test + public void testDefaultWarehouseeWithoutDatabaseLocation_asExpected() { + when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); + + assertEquals( + warehouseLocation + "/db.db/table", + bigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + } + + @Test + public void testCreateTable_succeedWhenNotExist() throws Exception { + // The table to create does not exist. + TableName tableName = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db", "tbl"); + TableIdentifier tableIdent = TableIdentifier.of("db", "tbl"); + Schema schema = BigLakeTestUtils.getTestSchema(); + + when(bigLakeClient.getTable(tableName)) + .thenThrow(new NoSuchTableException("error message getTable")); + Table createdTable = BigLakeTestUtils.createTestTable(tempFolder, bigLakeCatalog, tableName); + reset(bigLakeClient); + when(bigLakeClient.getTable(tableName)).thenReturn(createdTable, createdTable); + + org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(tableIdent); + assertEquals(SchemaParser.toJson(schema), SchemaParser.toJson(loadedTable.schema())); + + // Creates a table that already exists. + Exception exception = + assertThrows( + AlreadyExistsException.class, + () -> + bigLakeCatalog + .buildTable(tableIdent, schema) + .withLocation(tempFolder.newFolder("new_tbl").toString()) + .createTransaction() + .commitTransaction()); + assertTrue(exception.getMessage().contains("already exist")); + } + + @Test + public void testListTables_asExpected() { + when(bigLakeClient.listTables(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0"))) + .thenReturn( + ImmutableList.of( + Table.newBuilder() + .setName("projects/proj0/locations/us/catalogs/cat0/databases/db0/tables/tbl0") + .build(), + Table.newBuilder() + .setName("projects/proj0/locations/us/catalogs/cat0/databases/db0/tables/tbl1") + .build())); + + List result = bigLakeCatalog.listTables(Namespace.of("db0")); + assertEquals(2, result.size()); + assertEquals(TableIdentifier.of("db0", "tbl0"), result.get(0)); + assertEquals(TableIdentifier.of("db0", "tbl1"), result.get(1)); + } + + @Test + public void testListTables_emptyNamespace_checkCatalogEmptiness() { + when(bigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) + .thenReturn(ImmutableList.of(Database.getDefaultInstance())); + + List result = bigLakeCatalog.listTables(Namespace.of()); + assertEquals(1, result.size()); + assertEquals(TableIdentifier.of("placeholder"), result.get(0)); + } + + @Test + public void testListTables_emptyNamespace_noDatabase() { + when(bigLakeClient.listDatabases(any(CatalogName.class))).thenReturn(ImmutableList.of()); + + assertTrue(bigLakeCatalog.listTables(Namespace.of()).isEmpty()); + } + + @Test + public void testDropTable_throwWhenTableNotFound() { + TableName tableName = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db", "tbl"); + when(bigLakeClient.getTable(tableName)) + .thenThrow(new NoSuchTableException("error message getTable")); + doThrow(new NoSuchTableException("error message deleteTable")) + .when(bigLakeClient) + .deleteTable(tableName); + + Exception exception = + assertThrows( + NoSuchTableException.class, + () -> bigLakeCatalog.dropTable(TableIdentifier.of("db", "tbl"), /* purge = */ false)); + assertEquals("error message deleteTable", exception.getMessage()); + } + + @Test + public void testDropTable_succeedsWhenTableExists_deleteFiles() throws Exception { + TableName tableName = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db", "tbl"); + TableIdentifier tableIdent = TableIdentifier.of("db", "tbl"); + + when(bigLakeClient.getTable(tableName)) + .thenThrow(new NoSuchTableException("error message getTable")); + Table createdTable = BigLakeTestUtils.createTestTable(tempFolder, bigLakeCatalog, tableName); + String tableDir = createdTable.getHiveOptions().getStorageDescriptor().getLocationUri(); + assertTrue(BigLakeTestUtils.getIcebergMetadataFilePath(tableDir).isPresent()); + + reset(bigLakeClient); + when(bigLakeClient.getTable(tableName)).thenReturn(createdTable, createdTable); + when(bigLakeClient.deleteTable(tableName)).thenReturn(createdTable, createdTable); + + bigLakeCatalog.dropTable(tableIdent, /* purge = */ false); + assertTrue(BigLakeTestUtils.getIcebergMetadataFilePath(tableDir).isPresent()); + + bigLakeCatalog.dropTable(tableIdent, /* purge = */ true); + assertFalse(BigLakeTestUtils.getIcebergMetadataFilePath(tableDir).isPresent()); + } + + @Test + public void testRenameTable_sameDatabase_succeed() { + when(bigLakeClient.renameTable( + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t1"), + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t2"))) + .thenReturn( + Table.newBuilder() + .setName( + String.format( + "projects/%s/locations/us/catalogs/%s/databases/db0/tables/t2", + GCP_PROJECT, CATALOG_ID)) + .build()); + + bigLakeCatalog.renameTable(TableIdentifier.of("db0", "t1"), TableIdentifier.of("db0", "t2")); + } + + @Test + public void testRenameTable_differentDatabase_fail() { + when(bigLakeClient.renameTable( + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t1"), + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db1", "t2"))) + .thenReturn( + Table.newBuilder() + .setName( + String.format( + "projects/%s/locations/us/catalogs/%s/databases/db1/tables/t2", + GCP_PROJECT, CATALOG_ID)) + .build()); + + Exception exception = + assertThrows( + ValidationException.class, + () -> + bigLakeCatalog.renameTable( + TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); + assertEquals("New table name must be in the same database", exception.getMessage()); + } + + @Test + public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Exception { + when(bigLakeClient.createCatalog( + CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID), Catalog.getDefaultInstance())) + .thenReturn(Catalog.getDefaultInstance()); + + bigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + } + + @Test + public void testCreateNamespace_createDatabase() throws Exception { + String dbId = "db"; + Map metadata = ImmutableMap.of(); + String dbDir = warehouseLocation + String.format("/%s.db", dbId); + Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); + builder.getHiveOptionsBuilder().putAllParameters(metadata).setLocationUri(dbDir); + Database db = builder.build(); + when(bigLakeClient.createDatabase( + DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, dbId), db)) + .thenReturn(db); + + bigLakeCatalog.createNamespace(Namespace.of(new String[] {dbId}), metadata); + File f = new File(dbDir); + assertTrue(f.exists()); + assertTrue(f.isDirectory()); + } + + @Test + public void testCreateNamespace_failWhenInvalid() throws Exception { + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> + bigLakeCatalog.createNamespace( + Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())); + assertEquals( + "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + + " namespace: n0.n1", + exception.getMessage()); + } + + @Test + public void testListNamespaces_asExpected() { + when(bigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) + .thenReturn( + ImmutableList.of( + Database.newBuilder() + .setName("projects/proj0/locations/us/catalogs/cat0/databases/db0") + .build(), + Database.newBuilder() + .setName("projects/proj0/locations/us/catalogs/cat0/databases/db1") + .build())); + + List result = bigLakeCatalog.listNamespaces(Namespace.of()); + assertEquals(2, result.size()); + assertEquals(Namespace.of("db0"), result.get(0)); + assertEquals(Namespace.of("db1"), result.get(1)); + } + + @Test + public void testListNamespaces_emptyWhenInvalid() { + assertTrue(bigLakeCatalog.listNamespaces(Namespace.of("db")).isEmpty()); + } + + @Test + public void testDropNamespace_deleteCatalogWhenEmptyNamespace() { + doNothing() + .when(bigLakeClient) + .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); + + bigLakeCatalog.dropNamespace(Namespace.of(new String[] {})); + } + + @Test + public void testDropNamespace_deleteDatabase() { + doNothing() + .when(bigLakeClient) + .deleteDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db")); + + bigLakeCatalog.dropNamespace(Namespace.of(new String[] {"db"})); + } + + @Test + public void testDropNamespace_failWhenInvalid() throws Exception { + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> bigLakeCatalog.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))); + assertEquals( + "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + + " namespace: n0.n1", + exception.getMessage()); + } + + @Test + public void testSetProperties_failWhenNamespacesAreInvalid() throws Exception { + assertFalse(bigLakeCatalog.setProperties(Namespace.of(new String[] {}), ImmutableMap.of())); + assertFalse( + bigLakeCatalog.setProperties(Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); + } + + @Test + public void testSetProperties_succeedForDatabase() throws Exception { + when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions( + HiveDatabaseOptions.newBuilder() + .putParameters("key1", "value1") + .putParameters("key2", "value2")) + .build()); + when(bigLakeClient.updateDatabaseParameters( + DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), + ImmutableMap.of("key3", "value3"))) + .thenReturn(Database.getDefaultInstance()); + + assertTrue( + bigLakeCatalog.setProperties( + Namespace.of(new String[] {"db"}), + ImmutableMap.of("key1", "value1", "key2", "value2", "key3", "value3"))); + } + + @Test + public void testRemoveProperties_failWhenNamespacesAreInvalid() throws Exception { + assertFalse(bigLakeCatalog.removeProperties(Namespace.of(new String[] {}), ImmutableSet.of())); + assertFalse( + bigLakeCatalog.removeProperties( + Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())); + } + + @Test + public void testRemoveProperties_succeedForDatabase() throws Exception { + when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions( + HiveDatabaseOptions.newBuilder() + .putParameters("key1", "value1") + .putParameters("key2", "value2")) + .build()); + when(bigLakeClient.updateDatabaseParameters( + DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), + ImmutableMap.of("key2", "value2"))) + .thenReturn(Database.getDefaultInstance()); + + assertTrue( + bigLakeCatalog.removeProperties( + Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))); + } + + @Test + public void testLoadNamespaceMetadata_catalogAsExpected() throws Exception { + when(bigLakeClient.getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) + .thenReturn(Catalog.getDefaultInstance()); + + assertTrue(bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {})).isEmpty()); + } + + @Test + public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { + when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions( + HiveDatabaseOptions.newBuilder() + .setLocationUri("my location uri") + .putParameters("key1", "value1") + .putParameters("key2", "value2")) + .build()); + + assertEquals( + ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2"), + bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))); + } + + @Test + public void testLoadNamespaceMetadata_failWhenInvalid() throws Exception { + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"n0", "n1"}))); + assertEquals( + "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + + " namespace: n0.n1", + exception.getMessage()); + } + + @Test + public void testSetBigLakeCatalogInProperties_asExpected() throws Exception { + BigLakeCatalog catalog = new BigLakeCatalog(); + catalog.initialize( + CATALOG_ID, + /* properties= */ ImmutableMap.of( + BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, + GCP_PROJECT, + CatalogProperties.WAREHOUSE_LOCATION, + warehouseLocation, + BigLakeCatalog.PROPERTIES_KEY_BLMS_CATALOG, + "customized_catalog"), + GCP_PROJECT, + GCP_REGION, + bigLakeClient); + + when(bigLakeClient.createCatalog( + CatalogName.of(GCP_PROJECT, GCP_REGION, "customized_catalog"), + Catalog.getDefaultInstance())) + .thenReturn(Catalog.getDefaultInstance()); + + bigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + } + + @Test + public void testName_asExpected() throws Exception { + assertEquals("biglake", bigLakeCatalog.name()); + } + + @Test + public void testProperties_asExpected() throws Exception { + assertEquals( + ImmutableMap.of("gcp_project", GCP_PROJECT, "warehouse", warehouseLocation), + bigLakeCatalog.properties()); + } + + @Test + public void testNewTableOps_asExpected() throws Exception { + assertNotNull(bigLakeCatalog.newTableOps(TableIdentifier.of("db", "tbl"))); + } + + @Test + public void testNewTableOps_failedForInvalidNamespace() throws Exception { + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> bigLakeCatalog.newTableOps(TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))); + assertEquals( + "BigLake database namespace must use format ., invalid namespace: n0.n1", + exception.getMessage()); + } +} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java new file mode 100644 index 000000000000..4a01b13361a0 --- /dev/null +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.rpc.AbortedException; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import io.grpc.Status.Code; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +public class BigLakeTableOperationsTest { + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + + private static final String GCP_PROJECT = "my-project"; + private static final String GCP_REGION = "us"; + private static final String CATALOG_ID = "biglake"; + private static final String DB_ID = "db"; + private static final String TABLE_ID = "tbl"; + private static final TableName TABLE_NAME = + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, DB_ID, TABLE_ID); + private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DB_ID, TABLE_ID); + + @Mock private BigLakeClient bigLakeClient; + + private BigLakeCatalog bigLakeCatalog; + private String warehouseLocation; + private BigLakeTableOperations tableOps; + + @Before + public void before() throws Exception { + this.bigLakeCatalog = new BigLakeCatalog(); + this.warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + + bigLakeCatalog.initialize( + CATALOG_ID, + /* properties= */ ImmutableMap.of( + BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, + GCP_PROJECT, + CatalogProperties.WAREHOUSE_LOCATION, + warehouseLocation), + GCP_PROJECT, + GCP_REGION, + bigLakeClient); + this.tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(SPARK_TABLE_ID); + } + + @Test + public void testDoFresh_fetchLatestMetadataFromBigLake() throws Exception { + Table createdTable = createTestTable(); + reset(bigLakeClient); + when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(createdTable); + + tableOps.refresh(); + assertEquals( + createdTable + .getHiveOptions() + .getParametersOrDefault(BigLakeTestUtils.METADATA_LOCATION_PROP, ""), + tableOps.currentMetadataLocation()); + + reset(bigLakeClient); + when(bigLakeClient.getTable(TABLE_NAME)) + .thenThrow(new NoSuchTableException("error message getTable")); + // Refresh fails when table is not found but metadata already presents. + assertThrows(NoSuchTableException.class, () -> tableOps.refresh()); + } + + @Test + public void testDoFresh_failForNonIcebergTable() throws Exception { + when(bigLakeClient.getTable(TABLE_NAME)) + .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); + + Exception exception = assertThrows(ValidationException.class, () -> tableOps.refresh()); + assertTrue(exception.getMessage().contains("metadata location not found")); + } + + @Test + public void testDoFresh_noOpWhenMetadataAndTableNotFound() throws Exception { + when(bigLakeClient.getTable(TABLE_NAME)) + .thenThrow(new NoSuchTableException("error message getTable")); + // Table not found won't cause errors when the metadata is null. + assertEquals(null, tableOps.currentMetadataLocation()); + tableOps.refresh(); + } + + @Test + public void testTableName_asExpected() throws Exception { + assertEquals("biglake.db.tbl", tableOps.tableName()); + } + + @Test + public void testDoCommit_useEtagForUpdateTable() throws Exception { + Table createdTable = createTestTable(); + Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); + reset(bigLakeClient); + when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); + + org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(SPARK_TABLE_ID); + + when(bigLakeClient.updateTableParameters(any(), any(), any())).thenReturn(tableWithEtag); + loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit(); + + ArgumentCaptor nameCaptor = ArgumentCaptor.forClass(TableName.class); + ArgumentCaptor etagCaptor = ArgumentCaptor.forClass(String.class); + verify(bigLakeClient, times(1)) + .updateTableParameters(nameCaptor.capture(), any(), etagCaptor.capture()); + assertEquals(TABLE_NAME, nameCaptor.getValue()); + assertEquals("etag", etagCaptor.getValue()); + } + + @Test + public void testDoCommit_failWhenEtagMismatch() throws Exception { + Table createdTable = createTestTable(); + Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); + reset(bigLakeClient); + when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); + + org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(SPARK_TABLE_ID); + + when(bigLakeClient.updateTableParameters(any(), any(), any())) + .thenThrow( + new AbortedException( + new RuntimeException("error message etag mismatch"), + GrpcStatusCode.of(Code.ABORTED), + false)); + assertThrows( + CommitFailedException.class, + () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()); + } + + @Test + public void testDoCommit_failWhenMetadataLocationDiff() throws Exception { + Table createdTable = createTestTable(); + Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); + Table.Builder tableWithNewMetadata = tableWithEtag.toBuilder(); + tableWithNewMetadata + .getHiveOptionsBuilder() + .putParameters(BigLakeTestUtils.METADATA_LOCATION_PROP, "a new location"); + + reset(bigLakeClient); + // Two invocations, for loadTable and commit. + when(bigLakeClient.getTable(TABLE_NAME)) + .thenReturn(tableWithEtag, tableWithNewMetadata.build()); + + org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(SPARK_TABLE_ID); + + when(bigLakeClient.updateTableParameters(any(), any(), any())).thenReturn(tableWithEtag); + assertThrows( + CommitFailedException.class, + () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()); + } + + @Test + public void testCreateTable_doCommitSucceeds() throws Exception { + when(bigLakeClient.getTable(TABLE_NAME)) + .thenThrow(new NoSuchTableException("error message getTable")); + when(bigLakeClient.createTable(eq(TABLE_NAME), any())).thenReturn(Table.getDefaultInstance()); + when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + .thenReturn( + Database.newBuilder() + .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) + .build()); + + Schema schema = BigLakeTestUtils.getTestSchema(); + bigLakeCatalog.createTable(SPARK_TABLE_ID, schema, PartitionSpec.unpartitioned()); + } + + /** Creates a test table to have Iceberg metadata files in place. */ + private Table createTestTable() throws Exception { + when(bigLakeClient.getTable(TABLE_NAME)) + .thenThrow(new NoSuchTableException("error message getTable")); + return BigLakeTestUtils.createTestTable(tempFolder, bigLakeCatalog, TABLE_NAME); + } +} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java new file mode 100644 index 000000000000..263466c19deb --- /dev/null +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions.StorageDescriptor; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.File; +import java.io.IOException; +import java.util.Optional; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.iceberg.Schema; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Types; +import org.junit.rules.TemporaryFolder; + +/** Test utility methods for BigLake Iceberg catalog. */ +public final class BigLakeTestUtils { + + public static final String METADATA_LOCATION_PROP = "metadata_location"; + + public static Schema getTestSchema() { + return new Schema( + required(1, "id", Types.IntegerType.get(), "unique ID"), + required(2, "data", Types.StringType.get())); + } + + public static Table createTestTable( + TemporaryFolder tempFolder, BigLakeCatalog biglakeCatalog, TableName tableName) + throws IOException { + Schema schema = getTestSchema(); + TableIdentifier tableIdent = TableIdentifier.of(tableName.getDatabase(), tableName.getTable()); + String tableDir = tempFolder.newFolder(tableName.getTable()).toString(); + + biglakeCatalog + .buildTable(tableIdent, schema) + .withLocation(tableDir) + .createTransaction() + .commitTransaction(); + + Optional metadataLocation = getIcebergMetadataFilePath(tableDir); + assertTrue(metadataLocation.isPresent()); + return Table.newBuilder() + .setName(tableName.toString()) + .setHiveOptions( + HiveTableOptions.newBuilder() + .putParameters(METADATA_LOCATION_PROP, metadataLocation.get()) + .setStorageDescriptor(StorageDescriptor.newBuilder().setLocationUri(tableDir))) + .build(); + } + + public static Optional getIcebergMetadataFilePath(String tableDir) throws IOException { + for (File file : + FileUtils.listFiles(new File(tableDir), TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE)) { + if (file.getCanonicalPath().endsWith(".json")) { + return Optional.of(file.getCanonicalPath()); + } + } + return Optional.empty(); + } +} diff --git a/spark/v3.1/build.gradle b/spark/v3.1/build.gradle index bfb73ab9f230..7968f2e59902 100644 --- a/spark/v3.1/build.gradle +++ b/spark/v3.1/build.gradle @@ -216,6 +216,7 @@ project(':iceberg-spark:iceberg-spark-runtime-3.1_2.12') { implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } + implementation project(':iceberg-gcp') integrationImplementation "org.apache.spark:spark-hive_2.12:${sparkVersion}" integrationImplementation 'org.junit.vintage:junit-vintage-engine' diff --git a/spark/v3.2/build.gradle b/spark/v3.2/build.gradle index 2b57f49075c0..aa6a97067495 100644 --- a/spark/v3.2/build.gradle +++ b/spark/v3.2/build.gradle @@ -220,6 +220,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } + implementation project(':iceberg-gcp') integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}" diff --git a/spark/v3.3/build.gradle b/spark/v3.3/build.gradle index 875a7fe2ca51..39488a3dda8c 100644 --- a/spark/v3.3/build.gradle +++ b/spark/v3.3/build.gradle @@ -223,6 +223,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } + implementation project(':iceberg-gcp') integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}" diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle index bbd60f74b7d9..f073ec934121 100644 --- a/spark/v3.4/build.gradle +++ b/spark/v3.4/build.gradle @@ -223,6 +223,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } + implementation project(':iceberg-gcp') integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}" diff --git a/versions.props b/versions.props index 418efd6771ed..fd0e72ca27b8 100644 --- a/versions.props +++ b/versions.props @@ -30,6 +30,7 @@ com.emc.ecs:object-client-bundle = 3.3.2 org.immutables:value = 2.9.2 net.snowflake:snowflake-jdbc = 3.13.22 io.delta:delta-standalone_* = 0.6.0 +com.google.cloud:google-cloud-biglake = 0.3.0 # test deps org.junit.vintage:junit-vintage-engine = 5.9.2 From 4f1bce14a7aae891d0551bcda326e794472b3caa Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 25 Apr 2023 15:46:48 +0000 Subject: [PATCH 02/22] fix test errors in style check --- .../org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java | 6 +++--- .../org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 497519d1600a..8c6fbfc27be4 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -275,9 +275,9 @@ public void testCreateNamespace_createDatabase() throws Exception { .thenReturn(db); bigLakeCatalog.createNamespace(Namespace.of(new String[] {dbId}), metadata); - File f = new File(dbDir); - assertTrue(f.exists()); - assertTrue(f.isDirectory()); + File dir = new File(dbDir); + assertTrue(dir.exists()); + assertTrue(dir.isDirectory()); } @Test diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java index 263466c19deb..7130e8662391 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java @@ -79,4 +79,6 @@ public static Optional getIcebergMetadataFilePath(String tableDir) throw } return Optional.empty(); } + + private BigLakeTestUtils() {} } From 3a9948b0eec42af0e891dd311d7c739a7f6d6efb Mon Sep 17 00:00:00 2001 From: coufon Date: Fri, 28 Apr 2023 06:25:47 +0000 Subject: [PATCH 03/22] removed hadoop conf dependency, fix styles and tests --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 73 +++---------- .../iceberg/gcp/biglake/BigLakeClient.java | 2 +- .../gcp/biglake/BigLakeClientImpl.java | 2 +- .../gcp/biglake/BigLakeTableOperations.java | 30 +++--- .../gcp/biglake/BigLakeCatalogTest.java | 100 +++++++----------- .../biglake/BigLakeTableOperationsTest.java | 7 +- 6 files changed, 75 insertions(+), 139 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 0a57b906e4e2..c8750d71a78b 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -26,15 +26,10 @@ import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.iceberg.BaseMetastoreCatalog; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; @@ -44,9 +39,9 @@ import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.ServiceFailureException; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.hadoop.Util; +import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.ResolvingFileIO; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.base.Strings; @@ -61,7 +56,7 @@ /** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ public final class BigLakeCatalog extends BaseMetastoreCatalog - implements SupportsNamespaces, Configurable { + implements SupportsNamespaces, Configurable { // User provided properties. // The endpoint of BigLake API. Optional, default to DEFAULT_BIGLAKE_SERVICE_ENDPOINT. @@ -75,8 +70,6 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog // It links a BLMS catalog with this Iceberg catalog. public static final String PROPERTIES_KEY_BLMS_CATALOG = "blms_catalog"; - public static final String HIVE_METASTORE_WAREHOUSE_DIR = "hive.metastore.warehouse.dir"; - public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; public static final String DEFAULT_GCP_LOCATION = "us"; @@ -85,9 +78,8 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog // The name of this Iceberg catalog plugin: spark.sql.catalog.. private String catalogPulginName; private Map catalogProperties; - private FileSystem fs; private FileIO fileIO; - private Configuration conf; + private Object conf; private String projectId; private String location; // BLMS catalog ID and fully qualified name. @@ -102,9 +94,8 @@ public BigLakeCatalog() {} @Override public void initialize(String inputName, Map properties) { - if (!properties.containsKey(PROPERTIES_KEY_GCP_PROJECT)) { - throw new ValidationException("GCP project must be specified"); - } + Preconditions.checkArgument( + properties.containsKey(PROPERTIES_KEY_GCP_PROJECT), "GCP project must be specified"); String propProjectId = properties.get(PROPERTIES_KEY_GCP_PROJECT); String propLocation = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION); @@ -136,39 +127,19 @@ void initialize( Preconditions.checkNotNull(bigLakeClient, "BigLake client must not be null"); this.client = bigLakeClient; - if (this.conf == null) { - LOG.warn("No Hadoop Configuration was set, using the default environment Configuration"); - this.conf = new Configuration(); - } - // Users can specify the BigLake catalog ID, otherwise catalog plugin will be used. this.catalogId = properties.getOrDefault(PROPERTIES_KEY_BLMS_CATALOG, inputName); this.catalogName = CatalogName.of(projectId, location, catalogId); LOG.info("Use BigLake catalog: {}", catalogName.toString()); - if (properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION)) { - this.conf.set( - HIVE_METASTORE_WAREHOUSE_DIR, - LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION))); - } - - this.fs = - Util.getFs( - new Path( - LocationUtil.stripTrailingSlash( - properties.get(CatalogProperties.WAREHOUSE_LOCATION))), - conf); - String fileIOImpl = - properties.getOrDefault( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.hadoop.HadoopFileIO"); + properties.getOrDefault(CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); this.fileIO = CatalogUtil.loadFileIO(fileIOImpl, properties, conf); } @Override protected TableOperations newTableOps(TableIdentifier identifier) { return new BigLakeTableOperations( - conf, client, fileIO, getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); @@ -190,6 +161,7 @@ public List listTables(Namespace namespace) { // When deleting a BLMS catalog via `DROP NAMESPACE `, this method is called for // verifying catalog emptiness. `namespace` is empty in this case, we list databases in // this catalog instead. + // TODO: to return all tables in all databases in a BLMS catalog instead of a "placeholder". if (namespace.levels().length == 0) { return Iterables.isEmpty(client.listDatabases(catalogName)) ? ImmutableList.of() @@ -218,10 +190,8 @@ public void renameTable(TableIdentifier from, TableIdentifier to) { String fromDbId = getDatabaseId(from.namespace()); String toDbId = getDatabaseId(to.namespace()); - if (!fromDbId.equals(toDbId)) { - throw new ValidationException("New table name must be in the same database"); - } - + Preconditions.checkArgument( + fromDbId.equals(toDbId), "New table name must be in the same database"); client.renameTable(getTableName(fromDbId, from.name()), getTableName(toDbId, to.name())); } @@ -240,15 +210,7 @@ public void createNamespace(Namespace namespace, Map metadata) { .putAllParameters(metadata) .setLocationUri(getDatabaseLocation(dbId)); - Database db = - client.createDatabase( - DatabaseName.of(projectId, location, catalogId, dbId), builder.build()); - // Creates the data folder for the database. - try { - fs.mkdirs(new Path(db.getHiveOptions().getLocationUri())); - } catch (IOException e) { - throw new UncheckedIOException(String.format("Create namespace failed: %s", namespace), e); - } + client.createDatabase(DatabaseName.of(projectId, location, catalogId, dbId), builder.build()); } else { throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); } @@ -344,17 +306,14 @@ protected Map properties() { } @Override - public void setConf(Configuration conf) { - this.conf = new Configuration(conf); - } - - @Override - public Configuration getConf() { - return this.conf; + public void setConf(Object conf) { + this.conf = conf; } private String getDatabaseLocation(String dbId) { - String warehouseLocation = conf.get(HIVE_METASTORE_WAREHOUSE_DIR); + String warehouseLocation = + LocationUtil.stripTrailingSlash( + catalogProperties.get(CatalogProperties.WAREHOUSE_LOCATION)); Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set"); return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index 4373f4d2242a..73954f7a0abc 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -27,7 +27,7 @@ import java.util.Map; /** A client interface of Google BigLake service. */ -public interface BigLakeClient { +interface BigLakeClient { /** * Creates and returns a new catalog. diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java index 12cb9b257555..c37d07923d31 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java @@ -53,7 +53,7 @@ import org.apache.iceberg.exceptions.NotAuthorizedException; /** A client implementation of Google BigLake service. */ -public final class BigLakeClientImpl implements BigLakeClient { +final class BigLakeClientImpl implements BigLakeClient { private final String projectId; private final String location; diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index ed22724470b1..06842fdab842 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -25,7 +25,6 @@ import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; import java.util.Map; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.SnapshotSummary; @@ -34,8 +33,8 @@ import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,14 +44,11 @@ public final class BigLakeTableOperations extends BaseMetastoreTableOperations { private static final Logger LOG = LoggerFactory.getLogger(BigLakeTableOperations.class); - private final Configuration conf; private final BigLakeClient client; private final FileIO fileIO; private final TableName tableName; - BigLakeTableOperations( - Configuration conf, BigLakeClient client, FileIO fileIO, TableName tableName) { - this.conf = conf; + BigLakeTableOperations(BigLakeClient client, FileIO fileIO, TableName tableName) { this.client = client; this.fileIO = fileIO; this.tableName = tableName; @@ -65,10 +61,10 @@ public void doRefresh() { String metadataLocation = null; try { HiveTableOptions hiveOptions = client.getTable(tableName).getHiveOptions(); - if (!hiveOptions.containsParameters(METADATA_LOCATION_PROP)) { - throw new ValidationException( - "Table %s is not a valid Iceberg table, metadata location not found", tableName()); - } + Preconditions.checkArgument( + hiveOptions.containsParameters(METADATA_LOCATION_PROP), + "Table %s is not a valid Iceberg table, metadata location not found", + tableName()); metadataLocation = hiveOptions.getParametersOrThrow(METADATA_LOCATION_PROP); } catch (NoSuchTableException e) { if (currentMetadataLocation() != null) { @@ -141,12 +137,10 @@ private void updateTable( String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) { Table table = client.getTable(tableName); String etag = table.getEtag(); - if (etag.isEmpty()) { - throw new ValidationException( - "Etag of legacy table %s is empty, manually update the table by BigLake API or" - + " recreate and retry", - tableName()); - } + Preconditions.checkArgument( + !etag.isEmpty(), + "Etag of legacy table %s is empty, manually update the table by BigLake API or recreate and retry", + tableName()); HiveTableOptions options = table.getHiveOptions(); // If `metadataLocationFromMetastore` is different from metadata location of base, it means @@ -165,6 +159,10 @@ private void updateTable( } try { + // Updating a BLMS table with etag. The BLMS server checks that (1) the etag of a table on + // server is the same as the etag provided by the client and (2) update the table in the same + // transaction. The server returns an error containing message "etag mismatch", if the etag + // on server has changed. client.updateTableParameters( tableName, buildTableParameters(newMetadataLocation, metadata), etag); } catch (AbortedException e) { diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 8c6fbfc27be4..27a173bc5086 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -24,9 +24,10 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.cloud.bigquery.biglake.v1.Catalog; @@ -36,9 +37,9 @@ import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; -import java.io.File; import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.Schema; import org.apache.iceberg.SchemaParser; @@ -46,7 +47,6 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; @@ -54,6 +54,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; @@ -77,6 +78,7 @@ public void before() throws Exception { this.bigLakeCatalog = new BigLakeCatalog(); this.warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + bigLakeCatalog.setConf(new Configuration()); bigLakeCatalog.initialize( CATALOG_ID, /* properties= */ ImmutableMap.of( @@ -217,36 +219,18 @@ public void testDropTable_succeedsWhenTableExists_deleteFiles() throws Exception @Test public void testRenameTable_sameDatabase_succeed() { - when(bigLakeClient.renameTable( - TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t1"), - TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t2"))) - .thenReturn( - Table.newBuilder() - .setName( - String.format( - "projects/%s/locations/us/catalogs/%s/databases/db0/tables/t2", - GCP_PROJECT, CATALOG_ID)) - .build()); - bigLakeCatalog.renameTable(TableIdentifier.of("db0", "t1"), TableIdentifier.of("db0", "t2")); + verify(bigLakeClient, times(1)) + .renameTable( + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t1"), + TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t2")); } @Test public void testRenameTable_differentDatabase_fail() { - when(bigLakeClient.renameTable( - TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t1"), - TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db1", "t2"))) - .thenReturn( - Table.newBuilder() - .setName( - String.format( - "projects/%s/locations/us/catalogs/%s/databases/db1/tables/t2", - GCP_PROJECT, CATALOG_ID)) - .build()); - Exception exception = assertThrows( - ValidationException.class, + IllegalArgumentException.class, () -> bigLakeCatalog.renameTable( TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); @@ -255,11 +239,10 @@ public void testRenameTable_differentDatabase_fail() { @Test public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Exception { - when(bigLakeClient.createCatalog( - CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID), Catalog.getDefaultInstance())) - .thenReturn(Catalog.getDefaultInstance()); - bigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + verify(bigLakeClient, times(1)) + .createCatalog( + CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID), Catalog.getDefaultInstance()); } @Test @@ -269,15 +252,17 @@ public void testCreateNamespace_createDatabase() throws Exception { String dbDir = warehouseLocation + String.format("/%s.db", dbId); Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); builder.getHiveOptionsBuilder().putAllParameters(metadata).setLocationUri(dbDir); + + DatabaseName dbName = DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, dbId); Database db = builder.build(); - when(bigLakeClient.createDatabase( - DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, dbId), db)) - .thenReturn(db); + when(bigLakeClient.createDatabase(dbName, db)).thenReturn(db); bigLakeCatalog.createNamespace(Namespace.of(new String[] {dbId}), metadata); - File dir = new File(dbDir); - assertTrue(dir.exists()); - assertTrue(dir.isDirectory()); + ArgumentCaptor nameCaptor = ArgumentCaptor.forClass(DatabaseName.class); + ArgumentCaptor dbCaptor = ArgumentCaptor.forClass(Database.class); + verify(bigLakeClient, times(1)).createDatabase(nameCaptor.capture(), dbCaptor.capture()); + assertEquals(dbName, nameCaptor.getValue()); + assertEquals(db, dbCaptor.getValue()); } @Test @@ -319,20 +304,16 @@ public void testListNamespaces_emptyWhenInvalid() { @Test public void testDropNamespace_deleteCatalogWhenEmptyNamespace() { - doNothing() - .when(bigLakeClient) - .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); - bigLakeCatalog.dropNamespace(Namespace.of(new String[] {})); + verify(bigLakeClient, times(1)) + .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @Test public void testDropNamespace_deleteDatabase() { - doNothing() - .when(bigLakeClient) - .deleteDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db")); - bigLakeCatalog.dropNamespace(Namespace.of(new String[] {"db"})); + verify(bigLakeClient, times(1)) + .deleteDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db")); } @Test @@ -364,15 +345,15 @@ public void testSetProperties_succeedForDatabase() throws Exception { .putParameters("key1", "value1") .putParameters("key2", "value2")) .build()); - when(bigLakeClient.updateDatabaseParameters( - DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), - ImmutableMap.of("key3", "value3"))) - .thenReturn(Database.getDefaultInstance()); assertTrue( bigLakeCatalog.setProperties( Namespace.of(new String[] {"db"}), - ImmutableMap.of("key1", "value1", "key2", "value2", "key3", "value3"))); + ImmutableMap.of("key2", "value222", "key3", "value3"))); + verify(bigLakeClient, times(1)) + .updateDatabaseParameters( + DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), + ImmutableMap.of("key1", "value1", "key2", "value222", "key3", "value3")); } @Test @@ -393,22 +374,20 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { .putParameters("key1", "value1") .putParameters("key2", "value2")) .build()); - when(bigLakeClient.updateDatabaseParameters( - DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), - ImmutableMap.of("key2", "value2"))) - .thenReturn(Database.getDefaultInstance()); assertTrue( bigLakeCatalog.removeProperties( Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))); + verify(bigLakeClient, times(1)) + .updateDatabaseParameters( + DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), + ImmutableMap.of("key2", "value2")); } @Test public void testLoadNamespaceMetadata_catalogAsExpected() throws Exception { - when(bigLakeClient.getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) - .thenReturn(Catalog.getDefaultInstance()); - assertTrue(bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {})).isEmpty()); + verify(bigLakeClient, times(1)).getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @Test @@ -456,12 +435,11 @@ public void testSetBigLakeCatalogInProperties_asExpected() throws Exception { GCP_REGION, bigLakeClient); - when(bigLakeClient.createCatalog( + catalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + verify(bigLakeClient, times(1)) + .createCatalog( CatalogName.of(GCP_PROJECT, GCP_REGION, "customized_catalog"), - Catalog.getDefaultInstance())) - .thenReturn(Catalog.getDefaultInstance()); - - bigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + Catalog.getDefaultInstance()); } @Test diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 4a01b13361a0..d4871bcb9afb 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -36,13 +36,13 @@ import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; import io.grpc.Status.Code; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; import org.junit.Before; @@ -79,6 +79,7 @@ public void before() throws Exception { this.bigLakeCatalog = new BigLakeCatalog(); this.warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + bigLakeCatalog.setConf(new Configuration()); bigLakeCatalog.initialize( CATALOG_ID, /* properties= */ ImmutableMap.of( @@ -117,7 +118,7 @@ public void testDoFresh_failForNonIcebergTable() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); - Exception exception = assertThrows(ValidationException.class, () -> tableOps.refresh()); + Exception exception = assertThrows(IllegalArgumentException.class, () -> tableOps.refresh()); assertTrue(exception.getMessage().contains("metadata location not found")); } @@ -201,7 +202,6 @@ public void testDoCommit_failWhenMetadataLocationDiff() throws Exception { public void testCreateTable_doCommitSucceeds() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) .thenThrow(new NoSuchTableException("error message getTable")); - when(bigLakeClient.createTable(eq(TABLE_NAME), any())).thenReturn(Table.getDefaultInstance()); when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -210,6 +210,7 @@ public void testCreateTable_doCommitSucceeds() throws Exception { Schema schema = BigLakeTestUtils.getTestSchema(); bigLakeCatalog.createTable(SPARK_TABLE_ID, schema, PartitionSpec.unpartitioned()); + verify(bigLakeClient, times(1)).createTable(eq(TABLE_NAME), any()); } /** Creates a test table to have Iceberg metadata files in place. */ From c53b9b8fda79a1834c833fc8cfb5b557f80f14e4 Mon Sep 17 00:00:00 2001 From: coufon Date: Fri, 28 Apr 2023 16:18:02 +0000 Subject: [PATCH 04/22] update biglake config names --- .../apache/iceberg/gcp/biglake/BigLakeCatalog.java | 12 ++++++++---- .../iceberg/gcp/biglake/BigLakeClientImpl.java | 3 +-- .../iceberg/gcp/biglake/BigLakeTableOperations.java | 8 ++++---- .../iceberg/gcp/biglake/BigLakeCatalogTest.java | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index c8750d71a78b..9aee3ee31e3a 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -58,17 +58,18 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog implements SupportsNamespaces, Configurable { + // TODO: to move the configs to GCPProperties.java. // User provided properties. // The endpoint of BigLake API. Optional, default to DEFAULT_BIGLAKE_SERVICE_ENDPOINT. - public static final String PROPERTIES_KEY_BIGLAKE_ENDPOINT = "blms_endpoint"; + public static final String PROPERTIES_KEY_BIGLAKE_ENDPOINT = "biglake.endpoint"; // The GCP project ID. Required. - public static final String PROPERTIES_KEY_GCP_PROJECT = "gcp_project"; + public static final String PROPERTIES_KEY_GCP_PROJECT = "biglake.project-id"; // The GCP location (https://cloud.google.com/bigquery/docs/locations). Optional, default to // DEFAULT_GCP_LOCATION. - public static final String PROPERTIES_KEY_GCP_LOCATION = "gcp_location"; + public static final String PROPERTIES_KEY_GCP_LOCATION = "biglake.location"; // The BLMS catalog ID. It is the container resource of databases and tables. // It links a BLMS catalog with this Iceberg catalog. - public static final String PROPERTIES_KEY_BLMS_CATALOG = "blms_catalog"; + public static final String PROPERTIES_KEY_BLMS_CATALOG = "biglake.catalog"; public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; public static final String DEFAULT_GCP_LOCATION = "us"; @@ -101,6 +102,9 @@ public void initialize(String inputName, Map properties) { properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION); BigLakeClient newClient; try { + // TODO: to add more auth options of the client. Currently it uses default auth + // (https://github.com/googleapis/google-cloud-java#application-default-credentials) + // that works on GCP services (e.g., GCE, GKE, Dataproc). newClient = new BigLakeClientImpl( properties.getOrDefault( diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java index c37d07923d31..2c06b8dc5f64 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java @@ -66,8 +66,7 @@ final class BigLakeClientImpl implements BigLakeClient { * @param projectId GCP project ID * @param location GCP region supported by BigLake, e.g., "us" */ - public BigLakeClientImpl(String biglakeEndpoint, String projectId, String location) - throws IOException { + BigLakeClientImpl(String biglakeEndpoint, String projectId, String location) throws IOException { this.projectId = projectId; this.location = location; this.stub = diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 06842fdab842..7a13805e76b6 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -159,10 +159,10 @@ private void updateTable( } try { - // Updating a BLMS table with etag. The BLMS server checks that (1) the etag of a table on - // server is the same as the etag provided by the client and (2) update the table in the same - // transaction. The server returns an error containing message "etag mismatch", if the etag - // on server has changed. + // Updating a BLMS table with etag. The BLMS server transactionally (1) checks that the etag + // of a table on server is the same as the etag provided by the client, and (2) updates the + // table (and its etag). The server returns an error containing message "etag mismatch", if + // the etag on server has changed. client.updateTableParameters( tableName, buildTableParameters(newMetadataLocation, metadata), etag); } catch (AbortedException e) { diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 27a173bc5086..ae80241da05e 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -450,7 +450,7 @@ public void testName_asExpected() throws Exception { @Test public void testProperties_asExpected() throws Exception { assertEquals( - ImmutableMap.of("gcp_project", GCP_PROJECT, "warehouse", warehouseLocation), + ImmutableMap.of("biglake.project-id", GCP_PROJECT, "warehouse", warehouseLocation), bigLakeCatalog.properties()); } From 2d8dd2c100b0259b7b451d3d4885ee4cbdcb71e0 Mon Sep 17 00:00:00 2001 From: coufon Date: Thu, 11 May 2023 05:53:27 +0000 Subject: [PATCH 05/22] fix review comments --- .gitignore | 3 - build.gradle | 1 + .../iceberg/gcp/biglake/BigLakeCatalog.java | 45 ++- .../gcp/biglake/BigLakeClientImpl.java | 5 +- .../gcp/biglake/BigLakeTableOperations.java | 27 +- .../gcp/biglake/BigLakeCatalogTest.java | 340 ++++++------------ .../biglake/BigLakeTableOperationsTest.java | 151 +++----- .../iceberg/gcp/biglake/BigLakeTestUtils.java | 84 ----- .../gcp/biglake/FakeBigLakeClient.java | 179 +++++++++ spark/v3.1/build.gradle | 1 - spark/v3.2/build.gradle | 1 - spark/v3.3/build.gradle | 1 - spark/v3.4/build.gradle | 1 - 13 files changed, 399 insertions(+), 440 deletions(-) delete mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java create mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java diff --git a/.gitignore b/.gitignore index 5d1b337832f3..13e95b24648b 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,3 @@ derby.log python/.mypy_cache/ python/htmlcov python/coverage.xml - -# GCP test files -gcp/db_folder/ diff --git a/build.gradle b/build.gradle index 247fc1ec3a5f..824994b0f306 100644 --- a/build.gradle +++ b/build.gradle @@ -637,6 +637,7 @@ project(':iceberg-gcp') { testImplementation 'com.google.cloud:google-cloud-nio' testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') testImplementation("org.apache.hadoop:hadoop-common") { exclude group: 'org.apache.avro', module: 'avro' diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 9aee3ee31e3a..03cf5fef3a53 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -33,11 +33,14 @@ import org.apache.iceberg.BaseMetastoreCatalog; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.ServiceFailureException; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.FileIO; @@ -143,6 +146,14 @@ void initialize( @Override protected TableOperations newTableOps(TableIdentifier identifier) { + // The identifier of metadata tables is like "ns.table.files". + // We return a non-existing table in this case (empty table ID is disallowed in BigLake + // Metastore), loadTable will try loadMetadataTable. + if (identifier.namespace().levels().length > 1 + && MetadataTableType.from(identifier.name()) != null) { + return new BigLakeTableOperations( + client, fileIO, getTableName(identifier.namespace().level(0), /* tableId= */ "")); + } return new BigLakeTableOperations( client, fileIO, @@ -181,8 +192,13 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { TableOperations ops = newTableOps(identifier); // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. TableMetadata lastMetadata = ops.current(); - client.deleteTable( - getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); + try { + client.deleteTable( + getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); + } catch (NoSuchTableException e) { + LOG.warn("Dropping table failed", e); + return false; + } if (purge && lastMetadata != null) { CatalogUtil.dropTableData(ops.io(), lastMetadata); } @@ -235,16 +251,21 @@ public List listNamespaces(Namespace namespace) { @Override public boolean dropNamespace(Namespace namespace) { - if (namespace.levels().length == 0) { - // Used by `DROP NAMESPACE `. Deletes the BLMS catalog linked by Iceberg catalog. - client.deleteCatalog(catalogName); - LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); - } else if (namespace.levels().length == 1) { - client.deleteDatabase(getDatabaseName(namespace)); - // We don't delete the data file folder for safety. It aligns with HMS's default behavior. - // We can support database or catalog level config controlling file deletion in future. - } else { - throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + try { + if (namespace.levels().length == 0) { + // Used by `DROP NAMESPACE `. Deletes the BLMS catalog linked by Iceberg catalog. + client.deleteCatalog(catalogName); + LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); + } else if (namespace.levels().length == 1) { + client.deleteDatabase(getDatabaseName(namespace)); + // We don't delete the data file folder for safety. It aligns with HMS's default behavior. + // We can support database or catalog level config controlling file deletion in future. + } else { + throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + } + } catch (NoSuchNamespaceException e) { + LOG.warn("Dropping namespace failed", e); + return false; } return true; } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java index 2c06b8dc5f64..ac50ec72d23b 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java @@ -197,6 +197,9 @@ public Table createTable(TableName name, Table table) { @Override public Table getTable(TableName name) { + if (name.getTable().isEmpty()) { + throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); + } return convertException( () -> { try { @@ -271,7 +274,7 @@ private T convertException(Supplier result) { try { return result.get(); } catch (PermissionDeniedException e) { - throw new NotAuthorizedException(e, "Not authorized to call BigLake API"); + throw new NotAuthorizedException(e, "BigLake API permission denied"); } catch (com.google.api.gax.rpc.AlreadyExistsException e) { throw new AlreadyExistsException(e, "BigLake resource already exists"); } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 7a13805e76b6..b64e3160d0e8 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -30,8 +30,10 @@ import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -61,10 +63,10 @@ public void doRefresh() { String metadataLocation = null; try { HiveTableOptions hiveOptions = client.getTable(tableName).getHiveOptions(); - Preconditions.checkArgument( - hiveOptions.containsParameters(METADATA_LOCATION_PROP), - "Table %s is not a valid Iceberg table, metadata location not found", - tableName()); + if (!hiveOptions.containsParameters(METADATA_LOCATION_PROP)) { + throw new NoSuchIcebergTableException( + "Table %s is not a valid Iceberg table, metadata location not found", tableName()); + } metadataLocation = hiveOptions.getParametersOrThrow(METADATA_LOCATION_PROP); } catch (NoSuchTableException e) { if (currentMetadataLocation() != null) { @@ -90,7 +92,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { updateTable(base.metadataFileLocation(), newMetadataLocation, metadata); } commitStatus = CommitStatus.SUCCESS; - } catch (CommitFailedException | CommitStateUnknownException e) { + } catch (AlreadyExistsException | CommitFailedException | CommitStateUnknownException e) { throw e; } catch (Throwable e) { commitStatus = checkCommitStatus(newMetadataLocation, metadata); @@ -143,15 +145,19 @@ private void updateTable( tableName()); HiveTableOptions options = table.getHiveOptions(); - // If `metadataLocationFromMetastore` is different from metadata location of base, it means - // someone has updated metadata location in metastore, which is a conflict update. String metadataLocationFromMetastore = options.getParametersOrDefault(METADATA_LOCATION_PROP, ""); - if (!metadataLocationFromMetastore.isEmpty() - && !metadataLocationFromMetastore.equals(oldMetadataLocation)) { + if (metadataLocationFromMetastore.isEmpty()) { + throw new NoSuchIcebergTableException( + "Table %s is not a valid Iceberg table, metadata location is empty", tableName()); + } + // If `metadataLocationFromMetastore` is different from metadata location of base, it means + // someone has updated metadata location in metastore, which is a conflict update. + if (!metadataLocationFromMetastore.equals(oldMetadataLocation)) { throw new CommitFailedException( - "Base metadata location '%s' is not same as the current table metadata location '%s' for" + "Cannot commit %s. Base metadata location '%s' is not same as the current table metadata location '%s' for" + " %s.%s", + tableName(), oldMetadataLocation, metadataLocationFromMetastore, tableName.getDatabase(), @@ -170,6 +176,7 @@ private void updateTable( throw new CommitFailedException( "Updating table failed due to conflict updates (etag mismatch)"); } + throw e; } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index ae80241da05e..75737f93ef58 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -20,12 +20,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -35,65 +33,86 @@ import com.google.cloud.bigquery.biglake.v1.Database; import com.google.cloud.bigquery.biglake.v1.DatabaseName; import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; -import com.google.cloud.bigquery.biglake.v1.Table; -import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.File; +import java.nio.file.Path; import java.util.List; -import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.Schema; -import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; -import org.junit.Before; import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -public class BigLakeCatalogTest { +public class BigLakeCatalogTest extends CatalogTests { @Rule public final MockitoRule mockito = MockitoJUnit.rule(); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + @TempDir public Path temp; private static final String GCP_PROJECT = "my-project"; private static final String GCP_REGION = "us"; private static final String CATALOG_ID = "biglake"; - @Mock private BigLakeClient bigLakeClient; - - private BigLakeCatalog bigLakeCatalog; private String warehouseLocation; - @Before - public void before() throws Exception { - this.bigLakeCatalog = new BigLakeCatalog(); - this.warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + private BigLakeCatalog fakeBigLakeCatalog; - bigLakeCatalog.setConf(new Configuration()); - bigLakeCatalog.initialize( - CATALOG_ID, - /* properties= */ ImmutableMap.of( + private BigLakeClient mockBigLakeClient; + private BigLakeCatalog mockBigLakeCatalog; + + @BeforeEach + public void createCatalog() throws Exception { + File warehouse = temp.toFile(); + warehouseLocation = warehouse.getAbsolutePath(); + + ImmutableMap properties = + ImmutableMap.of( BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, - warehouseLocation), - GCP_PROJECT, - GCP_REGION, - bigLakeClient); + warehouseLocation); + + BigLakeClient fakeBigLakeClient = new FakeBigLakeClient(); + fakeBigLakeCatalog = new BigLakeCatalog(); + fakeBigLakeCatalog.setConf(new Configuration()); + fakeBigLakeCatalog.initialize( + CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, fakeBigLakeClient); + + mockBigLakeClient = mock(BigLakeClient.class); + mockBigLakeCatalog = new BigLakeCatalog(); + mockBigLakeCatalog.setConf(new Configuration()); + mockBigLakeCatalog.initialize( + CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, mockBigLakeClient); + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected BigLakeCatalog catalog() { + return fakeBigLakeCatalog; } + // By pass this test from CatalogTests, because BigLake API does not support "/" in resource IDs. + @Test + public void testNamespaceWithSlash() {} + + // By pass this test from CatalogTests, because BigLake API does not support "/" in resource IDs. + @Test + public void testTableNameWithSlash() {} + @Test public void testDefaultWarehouseWithDatabaseLocation_asExpected() { - when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) @@ -101,129 +120,18 @@ public void testDefaultWarehouseWithDatabaseLocation_asExpected() { assertEquals( "db_folder/table", - bigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + mockBigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); } @Test - public void testDefaultWarehouseeWithoutDatabaseLocation_asExpected() { - when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) + public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) .thenReturn( Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); assertEquals( warehouseLocation + "/db.db/table", - bigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); - } - - @Test - public void testCreateTable_succeedWhenNotExist() throws Exception { - // The table to create does not exist. - TableName tableName = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db", "tbl"); - TableIdentifier tableIdent = TableIdentifier.of("db", "tbl"); - Schema schema = BigLakeTestUtils.getTestSchema(); - - when(bigLakeClient.getTable(tableName)) - .thenThrow(new NoSuchTableException("error message getTable")); - Table createdTable = BigLakeTestUtils.createTestTable(tempFolder, bigLakeCatalog, tableName); - reset(bigLakeClient); - when(bigLakeClient.getTable(tableName)).thenReturn(createdTable, createdTable); - - org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(tableIdent); - assertEquals(SchemaParser.toJson(schema), SchemaParser.toJson(loadedTable.schema())); - - // Creates a table that already exists. - Exception exception = - assertThrows( - AlreadyExistsException.class, - () -> - bigLakeCatalog - .buildTable(tableIdent, schema) - .withLocation(tempFolder.newFolder("new_tbl").toString()) - .createTransaction() - .commitTransaction()); - assertTrue(exception.getMessage().contains("already exist")); - } - - @Test - public void testListTables_asExpected() { - when(bigLakeClient.listTables(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0"))) - .thenReturn( - ImmutableList.of( - Table.newBuilder() - .setName("projects/proj0/locations/us/catalogs/cat0/databases/db0/tables/tbl0") - .build(), - Table.newBuilder() - .setName("projects/proj0/locations/us/catalogs/cat0/databases/db0/tables/tbl1") - .build())); - - List result = bigLakeCatalog.listTables(Namespace.of("db0")); - assertEquals(2, result.size()); - assertEquals(TableIdentifier.of("db0", "tbl0"), result.get(0)); - assertEquals(TableIdentifier.of("db0", "tbl1"), result.get(1)); - } - - @Test - public void testListTables_emptyNamespace_checkCatalogEmptiness() { - when(bigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) - .thenReturn(ImmutableList.of(Database.getDefaultInstance())); - - List result = bigLakeCatalog.listTables(Namespace.of()); - assertEquals(1, result.size()); - assertEquals(TableIdentifier.of("placeholder"), result.get(0)); - } - - @Test - public void testListTables_emptyNamespace_noDatabase() { - when(bigLakeClient.listDatabases(any(CatalogName.class))).thenReturn(ImmutableList.of()); - - assertTrue(bigLakeCatalog.listTables(Namespace.of()).isEmpty()); - } - - @Test - public void testDropTable_throwWhenTableNotFound() { - TableName tableName = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db", "tbl"); - when(bigLakeClient.getTable(tableName)) - .thenThrow(new NoSuchTableException("error message getTable")); - doThrow(new NoSuchTableException("error message deleteTable")) - .when(bigLakeClient) - .deleteTable(tableName); - - Exception exception = - assertThrows( - NoSuchTableException.class, - () -> bigLakeCatalog.dropTable(TableIdentifier.of("db", "tbl"), /* purge = */ false)); - assertEquals("error message deleteTable", exception.getMessage()); - } - - @Test - public void testDropTable_succeedsWhenTableExists_deleteFiles() throws Exception { - TableName tableName = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db", "tbl"); - TableIdentifier tableIdent = TableIdentifier.of("db", "tbl"); - - when(bigLakeClient.getTable(tableName)) - .thenThrow(new NoSuchTableException("error message getTable")); - Table createdTable = BigLakeTestUtils.createTestTable(tempFolder, bigLakeCatalog, tableName); - String tableDir = createdTable.getHiveOptions().getStorageDescriptor().getLocationUri(); - assertTrue(BigLakeTestUtils.getIcebergMetadataFilePath(tableDir).isPresent()); - - reset(bigLakeClient); - when(bigLakeClient.getTable(tableName)).thenReturn(createdTable, createdTable); - when(bigLakeClient.deleteTable(tableName)).thenReturn(createdTable, createdTable); - - bigLakeCatalog.dropTable(tableIdent, /* purge = */ false); - assertTrue(BigLakeTestUtils.getIcebergMetadataFilePath(tableDir).isPresent()); - - bigLakeCatalog.dropTable(tableIdent, /* purge = */ true); - assertFalse(BigLakeTestUtils.getIcebergMetadataFilePath(tableDir).isPresent()); - } - - @Test - public void testRenameTable_sameDatabase_succeed() { - bigLakeCatalog.renameTable(TableIdentifier.of("db0", "t1"), TableIdentifier.of("db0", "t2")); - verify(bigLakeClient, times(1)) - .renameTable( - TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t1"), - TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db0", "t2")); + mockBigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); } @Test @@ -232,46 +140,26 @@ public void testRenameTable_differentDatabase_fail() { assertThrows( IllegalArgumentException.class, () -> - bigLakeCatalog.renameTable( + mockBigLakeCatalog.renameTable( TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); assertEquals("New table name must be in the same database", exception.getMessage()); } @Test public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Exception { - bigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); - verify(bigLakeClient, times(1)) + mockBigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + verify(mockBigLakeClient, times(1)) .createCatalog( CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID), Catalog.getDefaultInstance()); } - @Test - public void testCreateNamespace_createDatabase() throws Exception { - String dbId = "db"; - Map metadata = ImmutableMap.of(); - String dbDir = warehouseLocation + String.format("/%s.db", dbId); - Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); - builder.getHiveOptionsBuilder().putAllParameters(metadata).setLocationUri(dbDir); - - DatabaseName dbName = DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, dbId); - Database db = builder.build(); - when(bigLakeClient.createDatabase(dbName, db)).thenReturn(db); - - bigLakeCatalog.createNamespace(Namespace.of(new String[] {dbId}), metadata); - ArgumentCaptor nameCaptor = ArgumentCaptor.forClass(DatabaseName.class); - ArgumentCaptor dbCaptor = ArgumentCaptor.forClass(Database.class); - verify(bigLakeClient, times(1)).createDatabase(nameCaptor.capture(), dbCaptor.capture()); - assertEquals(dbName, nameCaptor.getValue()); - assertEquals(db, dbCaptor.getValue()); - } - @Test public void testCreateNamespace_failWhenInvalid() throws Exception { Exception exception = assertThrows( IllegalArgumentException.class, () -> - bigLakeCatalog.createNamespace( + mockBigLakeCatalog.createNamespace( Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())); assertEquals( "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" @@ -279,41 +167,38 @@ public void testCreateNamespace_failWhenInvalid() throws Exception { exception.getMessage()); } - @Test - public void testListNamespaces_asExpected() { - when(bigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) - .thenReturn( - ImmutableList.of( - Database.newBuilder() - .setName("projects/proj0/locations/us/catalogs/cat0/databases/db0") - .build(), - Database.newBuilder() - .setName("projects/proj0/locations/us/catalogs/cat0/databases/db1") - .build())); - - List result = bigLakeCatalog.listNamespaces(Namespace.of()); - assertEquals(2, result.size()); - assertEquals(Namespace.of("db0"), result.get(0)); - assertEquals(Namespace.of("db1"), result.get(1)); - } - @Test public void testListNamespaces_emptyWhenInvalid() { - assertTrue(bigLakeCatalog.listNamespaces(Namespace.of("db")).isEmpty()); + assertTrue(mockBigLakeCatalog.listNamespaces(Namespace.of("db")).isEmpty()); } @Test public void testDropNamespace_deleteCatalogWhenEmptyNamespace() { - bigLakeCatalog.dropNamespace(Namespace.of(new String[] {})); - verify(bigLakeClient, times(1)) + mockBigLakeCatalog.dropNamespace(Namespace.of(new String[] {})); + verify(mockBigLakeClient, times(1)) .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } + // BigLake catalog plugin supports dropping a BigLake catalog resource. Spark calls listTables + // with an empty namespace in this case, the purpose is verifying the namespace is empty. We + // check whether there are databases in the BigLake catalog instead. + @Test + public void testListTables_emptyNamespace_noDatabase() { + when(mockBigLakeClient.listDatabases(any(CatalogName.class))).thenReturn(ImmutableList.of()); + + assertTrue(mockBigLakeCatalog.listTables(Namespace.of()).isEmpty()); + verify(mockBigLakeClient, times(1)) + .listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); + } + @Test - public void testDropNamespace_deleteDatabase() { - bigLakeCatalog.dropNamespace(Namespace.of(new String[] {"db"})); - verify(bigLakeClient, times(1)) - .deleteDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db")); + public void testListTables_emptyNamespace_checkCatalogEmptiness() { + when(mockBigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) + .thenReturn(ImmutableList.of(Database.getDefaultInstance())); + + List result = mockBigLakeCatalog.listTables(Namespace.of()); + assertEquals(1, result.size()); + assertEquals(TableIdentifier.of("placeholder"), result.get(0)); } @Test @@ -321,7 +206,7 @@ public void testDropNamespace_failWhenInvalid() throws Exception { Exception exception = assertThrows( IllegalArgumentException.class, - () -> bigLakeCatalog.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))); + () -> mockBigLakeCatalog.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))); assertEquals( "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + " namespace: n0.n1", @@ -330,14 +215,15 @@ public void testDropNamespace_failWhenInvalid() throws Exception { @Test public void testSetProperties_failWhenNamespacesAreInvalid() throws Exception { - assertFalse(bigLakeCatalog.setProperties(Namespace.of(new String[] {}), ImmutableMap.of())); + assertFalse(mockBigLakeCatalog.setProperties(Namespace.of(new String[] {}), ImmutableMap.of())); assertFalse( - bigLakeCatalog.setProperties(Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); + mockBigLakeCatalog.setProperties( + Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); } @Test public void testSetProperties_succeedForDatabase() throws Exception { - when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions( @@ -347,10 +233,10 @@ public void testSetProperties_succeedForDatabase() throws Exception { .build()); assertTrue( - bigLakeCatalog.setProperties( + mockBigLakeCatalog.setProperties( Namespace.of(new String[] {"db"}), ImmutableMap.of("key2", "value222", "key3", "value3"))); - verify(bigLakeClient, times(1)) + verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), ImmutableMap.of("key1", "value1", "key2", "value222", "key3", "value3")); @@ -358,15 +244,16 @@ public void testSetProperties_succeedForDatabase() throws Exception { @Test public void testRemoveProperties_failWhenNamespacesAreInvalid() throws Exception { - assertFalse(bigLakeCatalog.removeProperties(Namespace.of(new String[] {}), ImmutableSet.of())); assertFalse( - bigLakeCatalog.removeProperties( + mockBigLakeCatalog.removeProperties(Namespace.of(new String[] {}), ImmutableSet.of())); + assertFalse( + mockBigLakeCatalog.removeProperties( Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())); } @Test public void testRemoveProperties_succeedForDatabase() throws Exception { - when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions( @@ -376,9 +263,9 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { .build()); assertTrue( - bigLakeCatalog.removeProperties( + mockBigLakeCatalog.removeProperties( Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))); - verify(bigLakeClient, times(1)) + verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), ImmutableMap.of("key2", "value2")); @@ -386,13 +273,14 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { @Test public void testLoadNamespaceMetadata_catalogAsExpected() throws Exception { - assertTrue(bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {})).isEmpty()); - verify(bigLakeClient, times(1)).getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); + assertTrue(mockBigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {})).isEmpty()); + verify(mockBigLakeClient, times(1)) + .getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @Test public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { - when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions( @@ -404,7 +292,7 @@ public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { assertEquals( ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2"), - bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))); + mockBigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))); } @Test @@ -412,7 +300,8 @@ public void testLoadNamespaceMetadata_failWhenInvalid() throws Exception { Exception exception = assertThrows( IllegalArgumentException.class, - () -> bigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"n0", "n1"}))); + () -> + mockBigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"n0", "n1"}))); assertEquals( "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + " namespace: n0.n1", @@ -433,38 +322,23 @@ public void testSetBigLakeCatalogInProperties_asExpected() throws Exception { "customized_catalog"), GCP_PROJECT, GCP_REGION, - bigLakeClient); + mockBigLakeClient); catalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); - verify(bigLakeClient, times(1)) + verify(mockBigLakeClient, times(1)) .createCatalog( CatalogName.of(GCP_PROJECT, GCP_REGION, "customized_catalog"), Catalog.getDefaultInstance()); } - @Test - public void testName_asExpected() throws Exception { - assertEquals("biglake", bigLakeCatalog.name()); - } - - @Test - public void testProperties_asExpected() throws Exception { - assertEquals( - ImmutableMap.of("biglake.project-id", GCP_PROJECT, "warehouse", warehouseLocation), - bigLakeCatalog.properties()); - } - - @Test - public void testNewTableOps_asExpected() throws Exception { - assertNotNull(bigLakeCatalog.newTableOps(TableIdentifier.of("db", "tbl"))); - } - @Test public void testNewTableOps_failedForInvalidNamespace() throws Exception { Exception exception = assertThrows( IllegalArgumentException.class, - () -> bigLakeCatalog.newTableOps(TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))); + () -> + mockBigLakeCatalog.newTableOps( + TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))); assertEquals( "BigLake database namespace must use format ., invalid namespace: n0.n1", exception.getMessage()); diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index d4871bcb9afb..88cb5573abc1 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -18,11 +18,11 @@ */ package org.apache.iceberg.gcp.biglake; +import static org.apache.iceberg.types.Types.NestedField.required; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.any; -import static org.mockito.Mockito.eq; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -30,15 +30,18 @@ import com.google.api.gax.grpc.GrpcStatusCode; import com.google.api.gax.rpc.AbortedException; -import com.google.cloud.bigquery.biglake.v1.Database; -import com.google.cloud.bigquery.biglake.v1.DatabaseName; -import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions; +import com.google.cloud.bigquery.biglake.v1.HiveTableOptions.StorageDescriptor; import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; import io.grpc.Status.Code; +import java.io.File; +import java.io.IOException; +import java.util.Optional; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; -import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; @@ -68,6 +71,11 @@ public class BigLakeTableOperationsTest { TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, DB_ID, TABLE_ID); private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DB_ID, TABLE_ID); + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.IntegerType.get(), "unique ID"), + required(2, "data", Types.StringType.get())); + @Mock private BigLakeClient bigLakeClient; private BigLakeCatalog bigLakeCatalog; @@ -76,69 +84,26 @@ public class BigLakeTableOperationsTest { @Before public void before() throws Exception { - this.bigLakeCatalog = new BigLakeCatalog(); - this.warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); - - bigLakeCatalog.setConf(new Configuration()); - bigLakeCatalog.initialize( - CATALOG_ID, - /* properties= */ ImmutableMap.of( + warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + ImmutableMap properties = + ImmutableMap.of( BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, - warehouseLocation), - GCP_PROJECT, - GCP_REGION, - bigLakeClient); - this.tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(SPARK_TABLE_ID); - } + warehouseLocation); - @Test - public void testDoFresh_fetchLatestMetadataFromBigLake() throws Exception { - Table createdTable = createTestTable(); - reset(bigLakeClient); - when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(createdTable); - - tableOps.refresh(); - assertEquals( - createdTable - .getHiveOptions() - .getParametersOrDefault(BigLakeTestUtils.METADATA_LOCATION_PROP, ""), - tableOps.currentMetadataLocation()); - - reset(bigLakeClient); - when(bigLakeClient.getTable(TABLE_NAME)) - .thenThrow(new NoSuchTableException("error message getTable")); - // Refresh fails when table is not found but metadata already presents. - assertThrows(NoSuchTableException.class, () -> tableOps.refresh()); - } - - @Test - public void testDoFresh_failForNonIcebergTable() throws Exception { - when(bigLakeClient.getTable(TABLE_NAME)) - .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); - - Exception exception = assertThrows(IllegalArgumentException.class, () -> tableOps.refresh()); - assertTrue(exception.getMessage().contains("metadata location not found")); + bigLakeCatalog = new BigLakeCatalog(); + bigLakeCatalog.setConf(new Configuration()); + bigLakeCatalog.initialize(CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, bigLakeClient); + this.tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(SPARK_TABLE_ID); } @Test - public void testDoFresh_noOpWhenMetadataAndTableNotFound() throws Exception { + public void testDoCommit_useEtagForUpdateTable() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) .thenThrow(new NoSuchTableException("error message getTable")); - // Table not found won't cause errors when the metadata is null. - assertEquals(null, tableOps.currentMetadataLocation()); - tableOps.refresh(); - } - - @Test - public void testTableName_asExpected() throws Exception { - assertEquals("biglake.db.tbl", tableOps.tableName()); - } - - @Test - public void testDoCommit_useEtagForUpdateTable() throws Exception { Table createdTable = createTestTable(); + Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); reset(bigLakeClient); when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); @@ -158,7 +123,10 @@ public void testDoCommit_useEtagForUpdateTable() throws Exception { @Test public void testDoCommit_failWhenEtagMismatch() throws Exception { + when(bigLakeClient.getTable(TABLE_NAME)) + .thenThrow(new NoSuchTableException("error message getTable")); Table createdTable = createTestTable(); + Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); reset(bigLakeClient); when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); @@ -177,46 +145,43 @@ public void testDoCommit_failWhenEtagMismatch() throws Exception { } @Test - public void testDoCommit_failWhenMetadataLocationDiff() throws Exception { - Table createdTable = createTestTable(); - Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); - Table.Builder tableWithNewMetadata = tableWithEtag.toBuilder(); - tableWithNewMetadata - .getHiveOptionsBuilder() - .putParameters(BigLakeTestUtils.METADATA_LOCATION_PROP, "a new location"); - - reset(bigLakeClient); - // Two invocations, for loadTable and commit. + public void testDoFresh_refreshReturnNullForNonIcebergTable() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) - .thenReturn(tableWithEtag, tableWithNewMetadata.build()); - - org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(SPARK_TABLE_ID); + .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); - when(bigLakeClient.updateTableParameters(any(), any(), any())).thenReturn(tableWithEtag); - assertThrows( - CommitFailedException.class, - () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()); + assertEquals(null, tableOps.refresh()); } - @Test - public void testCreateTable_doCommitSucceeds() throws Exception { - when(bigLakeClient.getTable(TABLE_NAME)) - .thenThrow(new NoSuchTableException("error message getTable")); - when(bigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) - .thenReturn( - Database.newBuilder() - .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) - .build()); - - Schema schema = BigLakeTestUtils.getTestSchema(); - bigLakeCatalog.createTable(SPARK_TABLE_ID, schema, PartitionSpec.unpartitioned()); - verify(bigLakeClient, times(1)).createTable(eq(TABLE_NAME), any()); + private Table createTestTable() throws IOException { + TableIdentifier tableIdent = + TableIdentifier.of(TABLE_NAME.getDatabase(), TABLE_NAME.getTable()); + String tableDir = tempFolder.newFolder(TABLE_NAME.getTable()).toString(); + + bigLakeCatalog + .buildTable(tableIdent, SCHEMA) + .withLocation(tableDir) + .createTransaction() + .commitTransaction(); + + Optional metadataLocation = getAnyIcebergMetadataFilePath(tableDir); + assertTrue(metadataLocation.isPresent()); + return Table.newBuilder() + .setName(TABLE_NAME.toString()) + .setHiveOptions( + HiveTableOptions.newBuilder() + .putParameters("metadata_location", metadataLocation.get()) + .setStorageDescriptor(StorageDescriptor.newBuilder().setLocationUri(tableDir))) + .build(); } - /** Creates a test table to have Iceberg metadata files in place. */ - private Table createTestTable() throws Exception { - when(bigLakeClient.getTable(TABLE_NAME)) - .thenThrow(new NoSuchTableException("error message getTable")); - return BigLakeTestUtils.createTestTable(tempFolder, bigLakeCatalog, TABLE_NAME); + private static Optional getAnyIcebergMetadataFilePath(String tableDir) + throws IOException { + for (File file : + FileUtils.listFiles(new File(tableDir), TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE)) { + if (file.getCanonicalPath().endsWith(".json")) { + return Optional.of(file.getCanonicalPath()); + } + } + return Optional.empty(); } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java deleted file mode 100644 index 7130e8662391..000000000000 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTestUtils.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.gcp.biglake; - -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.junit.Assert.assertTrue; - -import com.google.cloud.bigquery.biglake.v1.HiveTableOptions; -import com.google.cloud.bigquery.biglake.v1.HiveTableOptions.StorageDescriptor; -import com.google.cloud.bigquery.biglake.v1.Table; -import com.google.cloud.bigquery.biglake.v1.TableName; -import java.io.File; -import java.io.IOException; -import java.util.Optional; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.TrueFileFilter; -import org.apache.iceberg.Schema; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.types.Types; -import org.junit.rules.TemporaryFolder; - -/** Test utility methods for BigLake Iceberg catalog. */ -public final class BigLakeTestUtils { - - public static final String METADATA_LOCATION_PROP = "metadata_location"; - - public static Schema getTestSchema() { - return new Schema( - required(1, "id", Types.IntegerType.get(), "unique ID"), - required(2, "data", Types.StringType.get())); - } - - public static Table createTestTable( - TemporaryFolder tempFolder, BigLakeCatalog biglakeCatalog, TableName tableName) - throws IOException { - Schema schema = getTestSchema(); - TableIdentifier tableIdent = TableIdentifier.of(tableName.getDatabase(), tableName.getTable()); - String tableDir = tempFolder.newFolder(tableName.getTable()).toString(); - - biglakeCatalog - .buildTable(tableIdent, schema) - .withLocation(tableDir) - .createTransaction() - .commitTransaction(); - - Optional metadataLocation = getIcebergMetadataFilePath(tableDir); - assertTrue(metadataLocation.isPresent()); - return Table.newBuilder() - .setName(tableName.toString()) - .setHiveOptions( - HiveTableOptions.newBuilder() - .putParameters(METADATA_LOCATION_PROP, metadataLocation.get()) - .setStorageDescriptor(StorageDescriptor.newBuilder().setLocationUri(tableDir))) - .build(); - } - - public static Optional getIcebergMetadataFilePath(String tableDir) throws IOException { - for (File file : - FileUtils.listFiles(new File(tableDir), TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE)) { - if (file.getCanonicalPath().endsWith(".json")) { - return Optional.of(file.getCanonicalPath()); - } - } - return Optional.empty(); - } - - private BigLakeTestUtils() {} -} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java new file mode 100644 index 000000000000..266cd7c93b32 --- /dev/null +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; +import java.util.HashMap; +import java.util.Map; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; + +class FakeBigLakeClient implements BigLakeClient { + + private final Map catalogs; + private final Map dbs; + private final Map tables; + + FakeBigLakeClient() { + this.catalogs = new HashMap(); + this.dbs = new HashMap(); + this.tables = new HashMap(); + } + + @Override + public Catalog createCatalog(CatalogName name, Catalog catalog) { + if (catalogs.containsKey(name.toString())) { + throw new AlreadyExistsException("BigLake resource %s already exists", name.getCatalog()); + } + catalogs.put(name.toString(), catalog.toBuilder().setName(name.toString()).build()); + return catalog; + } + + @Override + public Catalog getCatalog(CatalogName name) { + if (catalogs.containsKey(name.toString())) { + return catalogs.get(name.toString()); + } + throw new NoSuchNamespaceException( + "Catalog %s does not exist or permission denied", name.toString()); + } + + @Override + public void deleteCatalog(CatalogName name) { + if (!catalogs.containsKey(name.toString())) { + throw new NoSuchNamespaceException( + "Catalog %s does not exist or permission denied", name.toString()); + } + } + + @Override + public Database createDatabase(DatabaseName name, Database db) { + if (dbs.containsKey(name.toString())) { + throw new AlreadyExistsException("BigLake resource %s already exists", name.getDatabase()); + } + dbs.put(name.toString(), db.toBuilder().setName(name.toString()).build()); + return db; + } + + @Override + public Database getDatabase(DatabaseName name) { + if (dbs.containsKey(name.toString())) { + return dbs.get(name.toString()); + } + throw new NoSuchNamespaceException("Namespace does not exist: %s", name.getDatabase()); + } + + @Override + public Database updateDatabaseParameters(DatabaseName name, Map parameters) { + if (!dbs.containsKey(name.toString())) { + throw new NoSuchNamespaceException( + "Database %s does not exist or permission denied", name.toString()); + } + Database.Builder dbBuilder = dbs.get(name.toString()).toBuilder(); + dbBuilder.getHiveOptionsBuilder().clearParameters().putAllParameters(parameters); + Database newDb = dbBuilder.build(); + dbs.put(name.toString(), newDb); + return newDb; + } + + @Override + public Iterable listDatabases(CatalogName name) { + return dbs.values(); + } + + @Override + public void deleteDatabase(DatabaseName name) { + if (!dbs.containsKey(name.toString())) { + throw new NoSuchNamespaceException( + "Database %s does not exist or permission denied", name.toString()); + } + dbs.remove(name.toString()); + } + + @Override + public Table createTable(TableName name, Table table) { + if (tables.containsKey(name.toString())) { + throw new AlreadyExistsException("Table already exists: %s", name.getTable()); + } + tables.put(name.toString(), table.toBuilder().setName(name.toString()).setEtag("etag").build()); + return table; + } + + @Override + public Table getTable(TableName name) { + if (name.getTable().isEmpty()) { + throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); + } + if (tables.containsKey(name.toString())) { + return tables.get(name.toString()); + } + throw new NoSuchTableException("Table %s does not exist or permission denied", name.toString()); + } + + @Override + public Table updateTableParameters(TableName name, Map parameters, String etag) { + if (!tables.containsKey(name.toString())) { + throw new NoSuchTableException( + "Table %s does not exist or permission denied", name.toString()); + } + Table.Builder tableBuilder = tables.get(name.toString()).toBuilder(); + tableBuilder.getHiveOptionsBuilder().clearParameters().putAllParameters(parameters); + Table newTable = tableBuilder.build(); + tables.put(name.toString(), newTable); + return newTable; + } + + @Override + public Table renameTable(TableName name, TableName newName) { + if (!tables.containsKey(name.toString())) { + throw new NoSuchTableException("Table does not exist or permission denied"); + } + if (tables.containsKey(newName.toString())) { + throw new AlreadyExistsException("Table already exists"); + } + Table table = tables.get(name.toString()); + Table newTable = table.toBuilder().setName(newName.toString()).build(); + tables.put(newName.toString(), newTable); + tables.remove(name.toString()); + return newTable; + } + + @Override + public Table deleteTable(TableName name) { + if (!tables.containsKey(name.toString())) { + throw new NoSuchTableException( + "Table %s does not exist or permission denied", name.toString()); + } + return tables.remove(name.toString()); + } + + @Override + public Iterable
listTables(DatabaseName name) { + return tables.values().stream() + .filter(t -> t.getName().contains(name.toString())) + .collect(ImmutableList.toImmutableList()); + } +} diff --git a/spark/v3.1/build.gradle b/spark/v3.1/build.gradle index 7968f2e59902..bfb73ab9f230 100644 --- a/spark/v3.1/build.gradle +++ b/spark/v3.1/build.gradle @@ -216,7 +216,6 @@ project(':iceberg-spark:iceberg-spark-runtime-3.1_2.12') { implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } - implementation project(':iceberg-gcp') integrationImplementation "org.apache.spark:spark-hive_2.12:${sparkVersion}" integrationImplementation 'org.junit.vintage:junit-vintage-engine' diff --git a/spark/v3.2/build.gradle b/spark/v3.2/build.gradle index aa6a97067495..2b57f49075c0 100644 --- a/spark/v3.2/build.gradle +++ b/spark/v3.2/build.gradle @@ -220,7 +220,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } - implementation project(':iceberg-gcp') integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}" diff --git a/spark/v3.3/build.gradle b/spark/v3.3/build.gradle index 39488a3dda8c..875a7fe2ca51 100644 --- a/spark/v3.3/build.gradle +++ b/spark/v3.3/build.gradle @@ -223,7 +223,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } - implementation project(':iceberg-gcp') integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}" diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle index f073ec934121..bbd60f74b7d9 100644 --- a/spark/v3.4/build.gradle +++ b/spark/v3.4/build.gradle @@ -223,7 +223,6 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio implementation (project(':iceberg-snowflake')) { exclude group: 'net.snowflake' , module: 'snowflake-jdbc' } - implementation project(':iceberg-gcp') integrationImplementation "org.scala-lang.modules:scala-collection-compat_${scalaVersion}" integrationImplementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}" From 5ddbb144baa3722583bcc83d1f1eb7604c995026 Mon Sep 17 00:00:00 2001 From: coufon Date: Mon, 15 May 2023 15:40:37 +0000 Subject: [PATCH 06/22] minor fix BigLake client error message --- .../iceberg/gcp/biglake/BigLakeClientImpl.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java index ac50ec72d23b..06c431690da4 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java @@ -94,7 +94,7 @@ public Catalog getCatalog(CatalogName name) { return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Catalog %s not found or permission denied", name.toString()); + e, "Catalog %s does not exist or permission denied", name.toString()); } }); } @@ -108,7 +108,7 @@ public void deleteCatalog(CatalogName name) { return Empty.getDefaultInstance(); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Catalog %s not found or permission denied", name.toString()); + e, "Catalog %s does not exist or permission denied", name.toString()); } }); } @@ -136,7 +136,7 @@ public Database getDatabase(DatabaseName name) { GetDatabaseRequest.newBuilder().setName(name.toString()).build()); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Database %s not found or permission denied", name.toString()); + e, "Database %s does not exist or permission denied", name.toString()); } }); } @@ -155,7 +155,7 @@ public Database updateDatabaseParameters(DatabaseName name, Map .build()); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Database %s not found or permission denied", name.toString()); + e, "Database %s does not exist or permission denied", name.toString()); } }); } @@ -178,7 +178,7 @@ public void deleteDatabase(DatabaseName name) { return Empty.getDefaultInstance(); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Database %s not found or permission denied", name.toString()); + e, "Database %s does not exist or permission denied", name.toString()); } }); } @@ -206,7 +206,7 @@ public Table getTable(TableName name) { return stub.getTable(GetTableRequest.newBuilder().setName(name.toString()).build()); } catch (PermissionDeniedException e) { throw new NoSuchTableException( - e, "Table %s not found or permission denied", name.toString()); + e, "Table %s does not exist or permission denied", name.toString()); } }); } @@ -225,7 +225,7 @@ public Table updateTableParameters(TableName name, Map parameter .build()); } catch (PermissionDeniedException e) { throw new NoSuchTableException( - e, "Table %s not found or permission denied", name.toString()); + e, "Table %s does not exist or permission denied", name.toString()); } }); } @@ -242,7 +242,7 @@ public Table renameTable(TableName name, TableName newName) { .build()); } catch (PermissionDeniedException e) { throw new NoSuchTableException( - e, "Table %s not found or permission denied", name.toString()); + e, "Table %s does not exist or permission denied", name.toString()); } }); } @@ -256,7 +256,7 @@ public Table deleteTable(TableName name) { DeleteTableRequest.newBuilder().setName(name.toString()).build()); } catch (PermissionDeniedException e) { throw new NoSuchTableException( - e, "Table %s not found or permission denied", name.toString()); + e, "Table %s does not exist or permission denied", name.toString()); } }); } From 3fa9e9e566b57fae74ac3137bb8ab090f3a6ea44 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 6 Jun 2023 15:20:00 +0000 Subject: [PATCH 07/22] add whitespace after control blocks --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 9 +++++++ .../gcp/biglake/BigLakeClientImpl.java | 3 ++- .../gcp/biglake/BigLakeTableOperations.java | 26 +++++-------------- .../gcp/biglake/FakeBigLakeClient.java | 6 +++-- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 03cf5fef3a53..8c4157598b04 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -103,6 +103,7 @@ public void initialize(String inputName, Map properties) { String propProjectId = properties.get(PROPERTIES_KEY_GCP_PROJECT); String propLocation = properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION); + BigLakeClient newClient; try { // TODO: to add more auth options of the client. Currently it uses default auth @@ -154,6 +155,7 @@ protected TableOperations newTableOps(TableIdentifier identifier) { return new BigLakeTableOperations( client, fileIO, getTableName(identifier.namespace().level(0), /* tableId= */ "")); } + return new BigLakeTableOperations( client, fileIO, @@ -182,6 +184,7 @@ public List listTables(Namespace namespace) { ? ImmutableList.of() : ImmutableList.of(TableIdentifier.of("placeholder")); } + return Streams.stream(client.listTables(getDatabaseName(namespace))) .map(BigLakeCatalog::getTableIdentifier) .collect(ImmutableList.toImmutableList()); @@ -199,9 +202,11 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { LOG.warn("Dropping table failed", e); return false; } + if (purge && lastMetadata != null) { CatalogUtil.dropTableData(ops.io(), lastMetadata); } + return true; } @@ -244,6 +249,7 @@ public List listNamespaces(Namespace namespace) { // well), returns empty to unblock deletion. return ImmutableList.of(); } + return Streams.stream(client.listDatabases(catalogName)) .map(BigLakeCatalog::getNamespace) .collect(ImmutableList.toImmutableList()); @@ -267,6 +273,7 @@ public boolean dropNamespace(Namespace namespace) { LOG.warn("Dropping namespace failed", e); return false; } + return true; } @@ -282,6 +289,7 @@ public boolean setProperties(Namespace namespace, Map properties e); return false; } + HiveDatabaseOptions.Builder optionsBuilder = builder.getHiveOptionsBuilder(); properties.forEach(optionsBuilder::putParameters); client.updateDatabaseParameters(getDatabaseName(namespace), optionsBuilder.getParametersMap()); @@ -301,6 +309,7 @@ public boolean removeProperties(Namespace namespace, Set properties) { e); return false; } + HiveDatabaseOptions.Builder optionsBuilder = builder.getHiveOptionsBuilder(); properties.forEach(optionsBuilder::removeParameters); client.updateDatabaseParameters(getDatabaseName(namespace), optionsBuilder.getParametersMap()); diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java index 06c431690da4..745ebb64f235 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java @@ -276,7 +276,8 @@ private T convertException(Supplier result) { } catch (PermissionDeniedException e) { throw new NotAuthorizedException(e, "BigLake API permission denied"); } catch (com.google.api.gax.rpc.AlreadyExistsException e) { - throw new AlreadyExistsException(e, "BigLake resource already exists"); + throw new AlreadyExistsException( + e, "Namespace already exists: BigLake resource already exists"); } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index b64e3160d0e8..56ec4f8d7e88 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -25,9 +25,7 @@ import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; import java.util.Map; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.iceberg.BaseMetastoreTableOperations; -import org.apache.iceberg.SnapshotSummary; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; import org.apache.iceberg.exceptions.AlreadyExistsException; @@ -67,6 +65,7 @@ public void doRefresh() { throw new NoSuchIcebergTableException( "Table %s is not a valid Iceberg table, metadata location not found", tableName()); } + metadataLocation = hiveOptions.getParametersOrThrow(METADATA_LOCATION_PROP); } catch (NoSuchTableException e) { if (currentMetadataLocation() != null) { @@ -74,6 +73,7 @@ public void doRefresh() { throw e; } } + refreshFromMetadataLocation(metadataLocation); } @@ -91,6 +91,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } else { updateTable(base.metadataFileLocation(), newMetadataLocation, metadata); } + commitStatus = CommitStatus.SUCCESS; } catch (AlreadyExistsException | CommitFailedException | CommitStateUnknownException e) { throw e; @@ -99,6 +100,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { if (commitStatus == CommitStatus.FAILURE) { throw new CommitFailedException(e, "Failed to commit"); } + if (commitStatus == CommitStatus.UNKNOWN) { throw new CommitStateUnknownException(e); } @@ -151,6 +153,7 @@ private void updateTable( throw new NoSuchIcebergTableException( "Table %s is not a valid Iceberg table, metadata location is empty", tableName()); } + // If `metadataLocationFromMetastore` is different from metadata location of base, it means // someone has updated metadata location in metastore, which is a conflict update. if (!metadataLocationFromMetastore.equals(oldMetadataLocation)) { @@ -205,30 +208,15 @@ private Map buildTableParameters( if (metadata.uuid() != null) { parameters.put(TableProperties.UUID, metadata.uuid()); } + if (currentMetadataLocation() != null && !currentMetadataLocation().isEmpty()) { parameters.put(PREVIOUS_METADATA_LOCATION_PROP, currentMetadataLocation()); } + parameters.put(METADATA_LOCATION_PROP, metadataFileLocation); // Follow HMS to use the EXTERNAL type. parameters.put("EXTERNAL", "TRUE"); parameters.put("table_type", "ICEBERG"); - - // Hive style basic statistics. - if (metadata.currentSnapshot() != null) { - Map summary = metadata.currentSnapshot().summary(); - if (summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP) != null) { - parameters.put( - StatsSetupConst.NUM_FILES, summary.get(SnapshotSummary.TOTAL_DATA_FILES_PROP)); - } - if (summary.get(SnapshotSummary.TOTAL_RECORDS_PROP) != null) { - parameters.put(StatsSetupConst.ROW_COUNT, summary.get(SnapshotSummary.TOTAL_RECORDS_PROP)); - } - if (summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP) != null) { - parameters.put( - StatsSetupConst.TOTAL_SIZE, summary.get(SnapshotSummary.TOTAL_FILE_SIZE_PROP)); - } - } - // TODO: to expose more Iceberg metadata if needed, e.g., statistic, schema, partition spec. return parameters; } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java index 266cd7c93b32..d050f39ef80b 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java @@ -46,7 +46,8 @@ class FakeBigLakeClient implements BigLakeClient { @Override public Catalog createCatalog(CatalogName name, Catalog catalog) { if (catalogs.containsKey(name.toString())) { - throw new AlreadyExistsException("BigLake resource %s already exists", name.getCatalog()); + throw new AlreadyExistsException( + "Namespace already exists: BigLake resource %s already exists", name.getCatalog()); } catalogs.put(name.toString(), catalog.toBuilder().setName(name.toString()).build()); return catalog; @@ -72,7 +73,8 @@ public void deleteCatalog(CatalogName name) { @Override public Database createDatabase(DatabaseName name, Database db) { if (dbs.containsKey(name.toString())) { - throw new AlreadyExistsException("BigLake resource %s already exists", name.getDatabase()); + throw new AlreadyExistsException( + "Namespace already exists: BigLake resource %s already exists", name.getDatabase()); } dbs.put(name.toString(), db.toBuilder().setName(name.toString()).build()); return db; From b38a82f6360e12693a0481dfb5e79534b731679f Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 6 Jun 2023 15:46:24 +0000 Subject: [PATCH 08/22] add negative tests of testNamespaceWithSlash and testTableNameWithSlash --- .../gcp/biglake/BigLakeCatalogTest.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 75737f93ef58..ef2b3087f588 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.api.pathtemplate.ValidationException; import com.google.cloud.bigquery.biglake.v1.Catalog; import com.google.cloud.bigquery.biglake.v1.CatalogName; import com.google.cloud.bigquery.biglake.v1.Database; @@ -102,13 +103,32 @@ protected BigLakeCatalog catalog() { return fakeBigLakeCatalog; } - // By pass this test from CatalogTests, because BigLake API does not support "/" in resource IDs. + @Override + protected boolean supportsNamesWithSlashes() { + return false; + } + @Test - public void testNamespaceWithSlash() {} + public void testNamespaceWithSlash() { + BigLakeCatalog catalog = catalog(); + + Exception exception = + assertThrows( + ValidationException.class, () -> catalog.createNamespace(Namespace.of("new/db"))); + assertEquals("Invalid character \"/\" in path section \"new/db\".", exception.getMessage()); + } - // By pass this test from CatalogTests, because BigLake API does not support "/" in resource IDs. @Test - public void testTableNameWithSlash() {} + public void testTableNameWithSlash() { + BigLakeCatalog catalog = catalog(); + + catalog.createNamespace(Namespace.of("ns")); + TableIdentifier ident = TableIdentifier.of("ns", "tab/le"); + + Exception exception = + assertThrows(ValidationException.class, () -> catalog.buildTable(ident, SCHEMA).create()); + assertEquals("Invalid character \"/\" in path section \"tab/le\".", exception.getMessage()); + } @Test public void testDefaultWarehouseWithDatabaseLocation_asExpected() { From 1be35ffa149ce7bb05307820c95e6aec6261ff47 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 6 Jun 2023 18:36:24 +0000 Subject: [PATCH 09/22] use mocked BigLake gRPC service --- build.gradle | 9 +- .../iceberg/gcp/biglake/BigLakeCatalog.java | 2 +- .../iceberg/gcp/biglake/BigLakeClient.java | 349 +++++++++++++----- .../gcp/biglake/BigLakeClientImpl.java | 291 --------------- .../gcp/biglake/BigLakeCatalogTest.java | 117 ++++-- .../gcp/biglake/FakeBigLakeClient.java | 181 --------- .../gcp/biglake/MockMetastoreService.java | 56 +++ .../gcp/biglake/MockMetastoreServiceImpl.java | 331 +++++++++++++++++ versions.props | 3 +- 9 files changed, 739 insertions(+), 600 deletions(-) delete mode 100644 gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java delete mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java create mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreService.java create mode 100644 gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreServiceImpl.java diff --git a/build.gradle b/build.gradle index dce40ff7d863..9af7d810b6a8 100644 --- a/build.gradle +++ b/build.gradle @@ -638,18 +638,19 @@ project(':iceberg-gcp') { exclude group: 'com.zaxxer', module: 'HikariCP' } - testImplementation 'com.google.cloud:google-cloud-nio' - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') - testImplementation("org.apache.hadoop:hadoop-common") { + testImplementation 'com.google.api.grpc:grpc-google-cloud-biglake-v1' + testImplementation 'com.google.cloud:google-cloud-nio' + testImplementation 'com.google.api:gax-grpc:2.28.1:testlib' + testImplementation 'com.esotericsoftware:kryo' + testImplementation('org.apache.hadoop:hadoop-common') { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'javax.servlet', module: 'servlet-api' exclude group: 'com.google.code.gson', module: 'gson' } - testImplementation "com.esotericsoftware:kryo" } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 8c4157598b04..f0f344b22252 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -110,7 +110,7 @@ public void initialize(String inputName, Map properties) { // (https://github.com/googleapis/google-cloud-java#application-default-credentials) // that works on GCP services (e.g., GCE, GKE, Dataproc). newClient = - new BigLakeClientImpl( + new BigLakeClient( properties.getOrDefault( PROPERTIES_KEY_BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), propProjectId, diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index 73954f7a0abc..a6fb344e4025 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -18,120 +18,293 @@ */ package org.apache.iceberg.gcp.biglake; +import com.google.api.gax.rpc.PermissionDeniedException; import com.google.cloud.bigquery.biglake.v1.Catalog; import com.google.cloud.bigquery.biglake.v1.CatalogName; +import com.google.cloud.bigquery.biglake.v1.CreateCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.CreateDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.CreateTableRequest; import com.google.cloud.bigquery.biglake.v1.Database; import com.google.cloud.bigquery.biglake.v1.DatabaseName; +import com.google.cloud.bigquery.biglake.v1.DeleteCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.DeleteDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.DeleteTableRequest; +import com.google.cloud.bigquery.biglake.v1.GetCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.GetDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.GetTableRequest; +import com.google.cloud.bigquery.biglake.v1.ListDatabasesRequest; +import com.google.cloud.bigquery.biglake.v1.ListTablesRequest; +import com.google.cloud.bigquery.biglake.v1.LocationName; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceClient; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import com.google.cloud.bigquery.biglake.v1.RenameTableRequest; import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; +import com.google.cloud.bigquery.biglake.v1.UpdateDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.UpdateTableRequest; +import com.google.protobuf.Empty; +import com.google.protobuf.FieldMask; +import java.io.IOException; import java.util.Map; +import java.util.function.Supplier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotAuthorizedException; -/** A client interface of Google BigLake service. */ -interface BigLakeClient { +/** A client of Google BigLake service. */ +final class BigLakeClient { - /** - * Creates and returns a new catalog. - * - * @param name full catalog name - * @param catalog body of catalog to create - */ - Catalog createCatalog(CatalogName name, Catalog catalog); + private final String projectId; + private final String location; + private final MetastoreServiceClient stub; /** - * Returns a catalog. + * Constructs a client of Google BigLake Service. * - * @param name full catalog name + * @param settings BigLake service settings + * @param projectId GCP project ID + * @param location GCP region supported by BigLake, e.g., "us" */ - Catalog getCatalog(CatalogName name); + BigLakeClient(MetastoreServiceSettings settings, String projectId, String location) + throws IOException { + this.projectId = projectId; + this.location = location; + this.stub = MetastoreServiceClient.create(settings); + } /** - * Deletes a catalog. + * Constructs a client of Google BigLake Service. * - * @param name full catalog name + * @param biglakeEndpoint BigLake service gRPC endpoint, e.g., "biglake.googleapis.com:443" + * @param projectId GCP project ID + * @param location GCP region supported by BigLake, e.g., "us" */ - void deleteCatalog(CatalogName name); + BigLakeClient(String biglakeEndpoint, String projectId, String location) throws IOException { + this( + MetastoreServiceSettings.newBuilder().setEndpoint(biglakeEndpoint).build(), + projectId, + location); + } - /** - * Creates and returns a new database. - * - * @param name full database name - * @param db body of database to create - */ - Database createDatabase(DatabaseName name, Database db); + public Catalog createCatalog(CatalogName name, Catalog catalog) { + return convertException( + () -> + stub.createCatalog( + CreateCatalogRequest.newBuilder() + .setParent(LocationName.of(name.getProject(), name.getLocation()).toString()) + .setCatalogId(name.getCatalog()) + .setCatalog(catalog) + .build()), + name.getCatalog()); + } - /** - * Returns a database. - * - * @param name full database name - */ - Database getDatabase(DatabaseName name); + public Catalog getCatalog(CatalogName name) { + return convertException( + () -> { + try { + return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s", name.getCatalog()); + } + }, + name.getCatalog()); + } - /** - * Updates the parameters of a Hive database and returns the updated database. - * - * @param name full database name - * @param parameters Hive options parameters to fully update - */ - Database updateDatabaseParameters(DatabaseName name, Map parameters); + public void deleteCatalog(CatalogName name) { + convertException( + () -> { + try { + stub.deleteCatalog(DeleteCatalogRequest.newBuilder().setName(name.toString()).build()); + return Empty.getDefaultInstance(); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s", name.getCatalog()); + } + }, + name.getCatalog()); + } - /** - * Returns all databases in a catalog. - * - * @param name full catalog name - */ - Iterable listDatabases(CatalogName name); + public Database createDatabase(DatabaseName name, Database db) { + return convertException( + () -> + stub.createDatabase( + CreateDatabaseRequest.newBuilder() + .setParent( + CatalogName.of(name.getProject(), name.getLocation(), name.getCatalog()) + .toString()) + .setDatabaseId(name.getDatabase()) + .setDatabase(db) + .build()), + name.getDatabase()); + } - /** - * Deletes a database. - * - * @param name full database name - */ - void deleteDatabase(DatabaseName name); + public Database getDatabase(DatabaseName name) { + return convertException( + () -> { + try { + return stub.getDatabase( + GetDatabaseRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s", name.getDatabase()); + } + }, + name.getDatabase()); + } - /** - * Creates and returns a new table. - * - * @param name full database name - * @param table body of table to create - */ - Table createTable(TableName name, Table table); + public Database updateDatabaseParameters(DatabaseName name, Map parameters) { + Database.Builder builder = Database.newBuilder().setName(name.toString()); + builder.getHiveOptionsBuilder().putAllParameters(parameters); + return convertException( + () -> { + try { + return stub.updateDatabase( + UpdateDatabaseRequest.newBuilder() + .setDatabase(builder) + .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s", name.getDatabase()); + } + }, + name.getDatabase()); + } - /** - * Returns a table. - * - * @param name full table name - */ - Table getTable(TableName name); + public Iterable listDatabases(CatalogName name) { + return convertException( + () -> + stub.listDatabases(ListDatabasesRequest.newBuilder().setParent(name.toString()).build()) + .iterateAll(), + name.getCatalog()); + } - /** - * Updates the parameters of a Hive table and returns the updated table. - * - * @param name full table name - * @param parameters Hive options parameters to fully update - * @param etag representation of table fields for concurrent update detection, see - * https://www.rfc-editor.org/rfc/rfc7232#section-2.3 - */ - Table updateTableParameters(TableName name, Map parameters, String etag); + public void deleteDatabase(DatabaseName name) { + convertException( + () -> { + try { + stub.deleteDatabase( + DeleteDatabaseRequest.newBuilder().setName(name.toString()).build()); + return Empty.getDefaultInstance(); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s", name.getDatabase()); + } + }, + name.getDatabase()); + } - /** - * Renames a table. - * - * @param name full table name - * @param newName new full table name - */ - Table renameTable(TableName name, TableName newName); + public Table createTable(TableName name, Table table) { + return convertException( + () -> { + try { + return stub.createTable( + CreateTableRequest.newBuilder() + .setParent(getDatabase(name).toString()) + .setTableId(name.getTable()) + .setTable(table) + .build()); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "Table already exists: %s", name.getTable()); + } + }, + name.getTable()); + } - /** - * Deletes a table. - * - * @param name full table name - */ - Table deleteTable(TableName name); + public Table getTable(TableName name) { + if (name.getTable().isEmpty()) { + throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); + } + return convertException( + () -> { + try { + return stub.getTable(GetTableRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } + }, + name.getTable()); + } - /** - * Returns all tables in a database. - * - * @param name full database name - */ - Iterable
listTables(DatabaseName name); + public Table updateTableParameters(TableName name, Map parameters, String etag) { + Table.Builder builder = Table.newBuilder().setName(name.toString()).setEtag(etag); + builder.getHiveOptionsBuilder().putAllParameters(parameters); + return convertException( + () -> { + try { + return stub.updateTable( + UpdateTableRequest.newBuilder() + .setTable(builder) + .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } + }, + name.getTable()); + } + + public Table renameTable(TableName name, TableName newName) { + return convertException( + () -> { + try { + return stub.renameTable( + RenameTableRequest.newBuilder() + .setName(name.toString()) + .setNewName(newName.toString()) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "Table already exists: %s", newName.getTable()); + } + }, + name.getTable()); + } + + public Table deleteTable(TableName name) { + return convertException( + () -> { + try { + return stub.deleteTable( + DeleteTableRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } + }, + name.getTable()); + } + + public Iterable
listTables(DatabaseName name) { + return convertException( + () -> + stub.listTables(ListTablesRequest.newBuilder().setParent(name.toString()).build()) + .iterateAll(), + name.getDatabase()); + } + + // Converts BigLake API errors to Iceberg errors. + private T convertException(Supplier result, String resourceId) { + try { + return result.get(); + } catch (PermissionDeniedException e) { + throw new NotAuthorizedException(e, "BigLake API permission denied"); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "Namespace already exists: %s", resourceId); + } + } + + private static DatabaseName getDatabase(TableName tableName) { + return DatabaseName.of( + tableName.getProject(), + tableName.getLocation(), + tableName.getCatalog(), + tableName.getDatabase()); + } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java deleted file mode 100644 index 745ebb64f235..000000000000 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClientImpl.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.gcp.biglake; - -import com.google.api.gax.rpc.PermissionDeniedException; -import com.google.cloud.bigquery.biglake.v1.Catalog; -import com.google.cloud.bigquery.biglake.v1.CatalogName; -import com.google.cloud.bigquery.biglake.v1.CreateCatalogRequest; -import com.google.cloud.bigquery.biglake.v1.CreateDatabaseRequest; -import com.google.cloud.bigquery.biglake.v1.CreateTableRequest; -import com.google.cloud.bigquery.biglake.v1.Database; -import com.google.cloud.bigquery.biglake.v1.DatabaseName; -import com.google.cloud.bigquery.biglake.v1.DeleteCatalogRequest; -import com.google.cloud.bigquery.biglake.v1.DeleteDatabaseRequest; -import com.google.cloud.bigquery.biglake.v1.DeleteTableRequest; -import com.google.cloud.bigquery.biglake.v1.GetCatalogRequest; -import com.google.cloud.bigquery.biglake.v1.GetDatabaseRequest; -import com.google.cloud.bigquery.biglake.v1.GetTableRequest; -import com.google.cloud.bigquery.biglake.v1.ListDatabasesRequest; -import com.google.cloud.bigquery.biglake.v1.ListTablesRequest; -import com.google.cloud.bigquery.biglake.v1.LocationName; -import com.google.cloud.bigquery.biglake.v1.MetastoreServiceClient; -import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; -import com.google.cloud.bigquery.biglake.v1.RenameTableRequest; -import com.google.cloud.bigquery.biglake.v1.Table; -import com.google.cloud.bigquery.biglake.v1.TableName; -import com.google.cloud.bigquery.biglake.v1.UpdateDatabaseRequest; -import com.google.cloud.bigquery.biglake.v1.UpdateTableRequest; -import com.google.protobuf.Empty; -import com.google.protobuf.FieldMask; -import java.io.IOException; -import java.util.Map; -import java.util.function.Supplier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NotAuthorizedException; - -/** A client implementation of Google BigLake service. */ -final class BigLakeClientImpl implements BigLakeClient { - - private final String projectId; - private final String location; - private final MetastoreServiceClient stub; - - /** - * Constructs a client of Google BigLake Service. - * - * @param biglakeEndpoint BigLake service gRPC endpoint, e.g., "biglake.googleapis.com:443" - * @param projectId GCP project ID - * @param location GCP region supported by BigLake, e.g., "us" - */ - BigLakeClientImpl(String biglakeEndpoint, String projectId, String location) throws IOException { - this.projectId = projectId; - this.location = location; - this.stub = - MetastoreServiceClient.create( - MetastoreServiceSettings.newBuilder().setEndpoint(biglakeEndpoint).build()); - } - - @Override - public Catalog createCatalog(CatalogName name, Catalog catalog) { - return convertException( - () -> - stub.createCatalog( - CreateCatalogRequest.newBuilder() - .setParent(LocationName.of(name.getProject(), name.getLocation()).toString()) - .setCatalogId(name.getCatalog()) - .setCatalog(catalog) - .build())); - } - - @Override - public Catalog getCatalog(CatalogName name) { - return convertException( - () -> { - try { - return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Catalog %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public void deleteCatalog(CatalogName name) { - convertException( - () -> { - try { - stub.deleteCatalog(DeleteCatalogRequest.newBuilder().setName(name.toString()).build()); - return Empty.getDefaultInstance(); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Catalog %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Database createDatabase(DatabaseName name, Database db) { - return convertException( - () -> - stub.createDatabase( - CreateDatabaseRequest.newBuilder() - .setParent( - CatalogName.of(name.getProject(), name.getLocation(), name.getCatalog()) - .toString()) - .setDatabaseId(name.getDatabase()) - .setDatabase(db) - .build())); - } - - @Override - public Database getDatabase(DatabaseName name) { - return convertException( - () -> { - try { - return stub.getDatabase( - GetDatabaseRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Database %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Database updateDatabaseParameters(DatabaseName name, Map parameters) { - Database.Builder builder = Database.newBuilder().setName(name.toString()); - builder.getHiveOptionsBuilder().putAllParameters(parameters); - return convertException( - () -> { - try { - return stub.updateDatabase( - UpdateDatabaseRequest.newBuilder() - .setDatabase(builder) - .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) - .build()); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Database %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Iterable listDatabases(CatalogName name) { - return convertException( - () -> - stub.listDatabases(ListDatabasesRequest.newBuilder().setParent(name.toString()).build()) - .iterateAll()); - } - - @Override - public void deleteDatabase(DatabaseName name) { - convertException( - () -> { - try { - stub.deleteDatabase( - DeleteDatabaseRequest.newBuilder().setName(name.toString()).build()); - return Empty.getDefaultInstance(); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Database %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Table createTable(TableName name, Table table) { - return convertException( - () -> - stub.createTable( - CreateTableRequest.newBuilder() - .setParent(getDatabase(name).toString()) - .setTableId(name.getTable()) - .setTable(table) - .build())); - } - - @Override - public Table getTable(TableName name) { - if (name.getTable().isEmpty()) { - throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); - } - return convertException( - () -> { - try { - return stub.getTable(GetTableRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Table updateTableParameters(TableName name, Map parameters, String etag) { - Table.Builder builder = Table.newBuilder().setName(name.toString()).setEtag(etag); - builder.getHiveOptionsBuilder().putAllParameters(parameters); - return convertException( - () -> { - try { - return stub.updateTable( - UpdateTableRequest.newBuilder() - .setTable(builder) - .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) - .build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Table renameTable(TableName name, TableName newName) { - return convertException( - () -> { - try { - return stub.renameTable( - RenameTableRequest.newBuilder() - .setName(name.toString()) - .setNewName(newName.toString()) - .build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Table deleteTable(TableName name) { - return convertException( - () -> { - try { - return stub.deleteTable( - DeleteTableRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table %s does not exist or permission denied", name.toString()); - } - }); - } - - @Override - public Iterable
listTables(DatabaseName name) { - return convertException( - () -> - stub.listTables(ListTablesRequest.newBuilder().setParent(name.toString()).build()) - .iterateAll()); - } - - // Converts BigLake API errors to Iceberg errors. - private T convertException(Supplier result) { - try { - return result.get(); - } catch (PermissionDeniedException e) { - throw new NotAuthorizedException(e, "BigLake API permission denied"); - } catch (com.google.api.gax.rpc.AlreadyExistsException e) { - throw new AlreadyExistsException( - e, "Namespace already exists: BigLake resource already exists"); - } - } - - private static DatabaseName getDatabase(TableName tableName) { - return DatabaseName.of( - tableName.getProject(), - tableName.getLocation(), - tableName.getCatalog(), - tableName.getDatabase()); - } -} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index ef2b3087f588..11ecb009c999 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -28,15 +28,23 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.grpc.testing.LocalChannelProvider; +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.api.gax.rpc.InvalidArgumentException; import com.google.api.pathtemplate.ValidationException; import com.google.cloud.bigquery.biglake.v1.Catalog; import com.google.cloud.bigquery.biglake.v1.CatalogName; import com.google.cloud.bigquery.biglake.v1.Database; import com.google.cloud.bigquery.biglake.v1.DatabaseName; import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; import java.io.File; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; +import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.CatalogTests; @@ -46,6 +54,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.junit.Rule; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -61,15 +71,35 @@ public class BigLakeCatalogTest extends CatalogTests { private static final String GCP_REGION = "us"; private static final String CATALOG_ID = "biglake"; - private String warehouseLocation; + private static MockMetastoreService mockMetastoreService; + private static MockServiceHelper mockServiceHelper; + + private LocalChannelProvider channelProvider; + private BigLakeCatalog bigLakeCatalogUsingMockService; - private BigLakeCatalog fakeBigLakeCatalog; + private String warehouseLocation; private BigLakeClient mockBigLakeClient; - private BigLakeCatalog mockBigLakeCatalog; + private BigLakeCatalog bigLakeCatalogUsingMockClient; + + @BeforeAll + public static void setUpStaticBigLakeService() throws Exception { + mockMetastoreService = new MockMetastoreService(); + mockServiceHelper = + new MockServiceHelper( + UUID.randomUUID().toString(), Arrays.asList(mockMetastoreService)); + mockServiceHelper.start(); + } + + @AfterAll + public static void stopStaticBigLakeService() { + mockServiceHelper.stop(); + } @BeforeEach public void createCatalog() throws Exception { + mockMetastoreService.reset(); + File warehouse = temp.toFile(); warehouseLocation = warehouse.getAbsolutePath(); @@ -80,16 +110,26 @@ public void createCatalog() throws Exception { CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); - BigLakeClient fakeBigLakeClient = new FakeBigLakeClient(); - fakeBigLakeCatalog = new BigLakeCatalog(); - fakeBigLakeCatalog.setConf(new Configuration()); - fakeBigLakeCatalog.initialize( - CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, fakeBigLakeClient); + channelProvider = mockServiceHelper.createChannelProvider(); + MetastoreServiceSettings settings = + MetastoreServiceSettings.newBuilder() + .setTransportChannelProvider(channelProvider) + .setCredentialsProvider(NoCredentialsProvider.create()) + .build(); + + bigLakeCatalogUsingMockService = new BigLakeCatalog(); + bigLakeCatalogUsingMockService.setConf(new Configuration()); + bigLakeCatalogUsingMockService.initialize( + CATALOG_ID, + properties, + GCP_PROJECT, + GCP_REGION, + new BigLakeClient(settings, GCP_PROJECT, GCP_REGION)); mockBigLakeClient = mock(BigLakeClient.class); - mockBigLakeCatalog = new BigLakeCatalog(); - mockBigLakeCatalog.setConf(new Configuration()); - mockBigLakeCatalog.initialize( + bigLakeCatalogUsingMockClient = new BigLakeCatalog(); + bigLakeCatalogUsingMockClient.setConf(new Configuration()); + bigLakeCatalogUsingMockClient.initialize( CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, mockBigLakeClient); } @@ -100,7 +140,7 @@ protected boolean requiresNamespaceCreate() { @Override protected BigLakeCatalog catalog() { - return fakeBigLakeCatalog; + return bigLakeCatalogUsingMockService; } @Override @@ -114,8 +154,8 @@ public void testNamespaceWithSlash() { Exception exception = assertThrows( - ValidationException.class, () -> catalog.createNamespace(Namespace.of("new/db"))); - assertEquals("Invalid character \"/\" in path section \"new/db\".", exception.getMessage()); + InvalidArgumentException.class, () -> catalog.createNamespace(Namespace.of("new/db"))); + assertTrue(exception.getMessage().contains("Database ID is invalid")); } @Test @@ -140,7 +180,7 @@ public void testDefaultWarehouseWithDatabaseLocation_asExpected() { assertEquals( "db_folder/table", - mockBigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); } @Test @@ -151,7 +191,7 @@ public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { assertEquals( warehouseLocation + "/db.db/table", - mockBigLakeCatalog.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); } @Test @@ -160,14 +200,14 @@ public void testRenameTable_differentDatabase_fail() { assertThrows( IllegalArgumentException.class, () -> - mockBigLakeCatalog.renameTable( + bigLakeCatalogUsingMockClient.renameTable( TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); assertEquals("New table name must be in the same database", exception.getMessage()); } @Test public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Exception { - mockBigLakeCatalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + bigLakeCatalogUsingMockClient.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); verify(mockBigLakeClient, times(1)) .createCatalog( CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID), Catalog.getDefaultInstance()); @@ -179,7 +219,7 @@ public void testCreateNamespace_failWhenInvalid() throws Exception { assertThrows( IllegalArgumentException.class, () -> - mockBigLakeCatalog.createNamespace( + bigLakeCatalogUsingMockClient.createNamespace( Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())); assertEquals( "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" @@ -189,12 +229,12 @@ public void testCreateNamespace_failWhenInvalid() throws Exception { @Test public void testListNamespaces_emptyWhenInvalid() { - assertTrue(mockBigLakeCatalog.listNamespaces(Namespace.of("db")).isEmpty()); + assertTrue(bigLakeCatalogUsingMockClient.listNamespaces(Namespace.of("db")).isEmpty()); } @Test public void testDropNamespace_deleteCatalogWhenEmptyNamespace() { - mockBigLakeCatalog.dropNamespace(Namespace.of(new String[] {})); + bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {})); verify(mockBigLakeClient, times(1)) .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @@ -206,7 +246,7 @@ public void testDropNamespace_deleteCatalogWhenEmptyNamespace() { public void testListTables_emptyNamespace_noDatabase() { when(mockBigLakeClient.listDatabases(any(CatalogName.class))).thenReturn(ImmutableList.of()); - assertTrue(mockBigLakeCatalog.listTables(Namespace.of()).isEmpty()); + assertTrue(bigLakeCatalogUsingMockClient.listTables(Namespace.of()).isEmpty()); verify(mockBigLakeClient, times(1)) .listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @@ -216,7 +256,7 @@ public void testListTables_emptyNamespace_checkCatalogEmptiness() { when(mockBigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) .thenReturn(ImmutableList.of(Database.getDefaultInstance())); - List result = mockBigLakeCatalog.listTables(Namespace.of()); + List result = bigLakeCatalogUsingMockClient.listTables(Namespace.of()); assertEquals(1, result.size()); assertEquals(TableIdentifier.of("placeholder"), result.get(0)); } @@ -226,7 +266,9 @@ public void testDropNamespace_failWhenInvalid() throws Exception { Exception exception = assertThrows( IllegalArgumentException.class, - () -> mockBigLakeCatalog.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))); + () -> + bigLakeCatalogUsingMockClient.dropNamespace( + Namespace.of(new String[] {"n0", "n1"}))); assertEquals( "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + " namespace: n0.n1", @@ -235,9 +277,11 @@ public void testDropNamespace_failWhenInvalid() throws Exception { @Test public void testSetProperties_failWhenNamespacesAreInvalid() throws Exception { - assertFalse(mockBigLakeCatalog.setProperties(Namespace.of(new String[] {}), ImmutableMap.of())); assertFalse( - mockBigLakeCatalog.setProperties( + bigLakeCatalogUsingMockClient.setProperties( + Namespace.of(new String[] {}), ImmutableMap.of())); + assertFalse( + bigLakeCatalogUsingMockClient.setProperties( Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); } @@ -253,7 +297,7 @@ public void testSetProperties_succeedForDatabase() throws Exception { .build()); assertTrue( - mockBigLakeCatalog.setProperties( + bigLakeCatalogUsingMockClient.setProperties( Namespace.of(new String[] {"db"}), ImmutableMap.of("key2", "value222", "key3", "value3"))); verify(mockBigLakeClient, times(1)) @@ -265,9 +309,10 @@ public void testSetProperties_succeedForDatabase() throws Exception { @Test public void testRemoveProperties_failWhenNamespacesAreInvalid() throws Exception { assertFalse( - mockBigLakeCatalog.removeProperties(Namespace.of(new String[] {}), ImmutableSet.of())); + bigLakeCatalogUsingMockClient.removeProperties( + Namespace.of(new String[] {}), ImmutableSet.of())); assertFalse( - mockBigLakeCatalog.removeProperties( + bigLakeCatalogUsingMockClient.removeProperties( Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())); } @@ -283,7 +328,7 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { .build()); assertTrue( - mockBigLakeCatalog.removeProperties( + bigLakeCatalogUsingMockClient.removeProperties( Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))); verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( @@ -293,7 +338,10 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { @Test public void testLoadNamespaceMetadata_catalogAsExpected() throws Exception { - assertTrue(mockBigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {})).isEmpty()); + assertTrue( + bigLakeCatalogUsingMockClient + .loadNamespaceMetadata(Namespace.of(new String[] {})) + .isEmpty()); verify(mockBigLakeClient, times(1)) .getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @@ -312,7 +360,7 @@ public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { assertEquals( ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2"), - mockBigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))); + bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))); } @Test @@ -321,7 +369,8 @@ public void testLoadNamespaceMetadata_failWhenInvalid() throws Exception { assertThrows( IllegalArgumentException.class, () -> - mockBigLakeCatalog.loadNamespaceMetadata(Namespace.of(new String[] {"n0", "n1"}))); + bigLakeCatalogUsingMockClient.loadNamespaceMetadata( + Namespace.of(new String[] {"n0", "n1"}))); assertEquals( "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" + " namespace: n0.n1", @@ -357,7 +406,7 @@ public void testNewTableOps_failedForInvalidNamespace() throws Exception { assertThrows( IllegalArgumentException.class, () -> - mockBigLakeCatalog.newTableOps( + bigLakeCatalogUsingMockClient.newTableOps( TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))); assertEquals( "BigLake database namespace must use format ., invalid namespace: n0.n1", diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java deleted file mode 100644 index d050f39ef80b..000000000000 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/FakeBigLakeClient.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.gcp.biglake; - -import com.google.cloud.bigquery.biglake.v1.Catalog; -import com.google.cloud.bigquery.biglake.v1.CatalogName; -import com.google.cloud.bigquery.biglake.v1.Database; -import com.google.cloud.bigquery.biglake.v1.DatabaseName; -import com.google.cloud.bigquery.biglake.v1.Table; -import com.google.cloud.bigquery.biglake.v1.TableName; -import java.util.HashMap; -import java.util.Map; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; - -class FakeBigLakeClient implements BigLakeClient { - - private final Map catalogs; - private final Map dbs; - private final Map tables; - - FakeBigLakeClient() { - this.catalogs = new HashMap(); - this.dbs = new HashMap(); - this.tables = new HashMap(); - } - - @Override - public Catalog createCatalog(CatalogName name, Catalog catalog) { - if (catalogs.containsKey(name.toString())) { - throw new AlreadyExistsException( - "Namespace already exists: BigLake resource %s already exists", name.getCatalog()); - } - catalogs.put(name.toString(), catalog.toBuilder().setName(name.toString()).build()); - return catalog; - } - - @Override - public Catalog getCatalog(CatalogName name) { - if (catalogs.containsKey(name.toString())) { - return catalogs.get(name.toString()); - } - throw new NoSuchNamespaceException( - "Catalog %s does not exist or permission denied", name.toString()); - } - - @Override - public void deleteCatalog(CatalogName name) { - if (!catalogs.containsKey(name.toString())) { - throw new NoSuchNamespaceException( - "Catalog %s does not exist or permission denied", name.toString()); - } - } - - @Override - public Database createDatabase(DatabaseName name, Database db) { - if (dbs.containsKey(name.toString())) { - throw new AlreadyExistsException( - "Namespace already exists: BigLake resource %s already exists", name.getDatabase()); - } - dbs.put(name.toString(), db.toBuilder().setName(name.toString()).build()); - return db; - } - - @Override - public Database getDatabase(DatabaseName name) { - if (dbs.containsKey(name.toString())) { - return dbs.get(name.toString()); - } - throw new NoSuchNamespaceException("Namespace does not exist: %s", name.getDatabase()); - } - - @Override - public Database updateDatabaseParameters(DatabaseName name, Map parameters) { - if (!dbs.containsKey(name.toString())) { - throw new NoSuchNamespaceException( - "Database %s does not exist or permission denied", name.toString()); - } - Database.Builder dbBuilder = dbs.get(name.toString()).toBuilder(); - dbBuilder.getHiveOptionsBuilder().clearParameters().putAllParameters(parameters); - Database newDb = dbBuilder.build(); - dbs.put(name.toString(), newDb); - return newDb; - } - - @Override - public Iterable listDatabases(CatalogName name) { - return dbs.values(); - } - - @Override - public void deleteDatabase(DatabaseName name) { - if (!dbs.containsKey(name.toString())) { - throw new NoSuchNamespaceException( - "Database %s does not exist or permission denied", name.toString()); - } - dbs.remove(name.toString()); - } - - @Override - public Table createTable(TableName name, Table table) { - if (tables.containsKey(name.toString())) { - throw new AlreadyExistsException("Table already exists: %s", name.getTable()); - } - tables.put(name.toString(), table.toBuilder().setName(name.toString()).setEtag("etag").build()); - return table; - } - - @Override - public Table getTable(TableName name) { - if (name.getTable().isEmpty()) { - throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); - } - if (tables.containsKey(name.toString())) { - return tables.get(name.toString()); - } - throw new NoSuchTableException("Table %s does not exist or permission denied", name.toString()); - } - - @Override - public Table updateTableParameters(TableName name, Map parameters, String etag) { - if (!tables.containsKey(name.toString())) { - throw new NoSuchTableException( - "Table %s does not exist or permission denied", name.toString()); - } - Table.Builder tableBuilder = tables.get(name.toString()).toBuilder(); - tableBuilder.getHiveOptionsBuilder().clearParameters().putAllParameters(parameters); - Table newTable = tableBuilder.build(); - tables.put(name.toString(), newTable); - return newTable; - } - - @Override - public Table renameTable(TableName name, TableName newName) { - if (!tables.containsKey(name.toString())) { - throw new NoSuchTableException("Table does not exist or permission denied"); - } - if (tables.containsKey(newName.toString())) { - throw new AlreadyExistsException("Table already exists"); - } - Table table = tables.get(name.toString()); - Table newTable = table.toBuilder().setName(newName.toString()).build(); - tables.put(newName.toString(), newTable); - tables.remove(name.toString()); - return newTable; - } - - @Override - public Table deleteTable(TableName name) { - if (!tables.containsKey(name.toString())) { - throw new NoSuchTableException( - "Table %s does not exist or permission denied", name.toString()); - } - return tables.remove(name.toString()); - } - - @Override - public Iterable
listTables(DatabaseName name) { - return tables.values().stream() - .filter(t -> t.getName().contains(name.toString())) - .collect(ImmutableList.toImmutableList()); - } -} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreService.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreService.java new file mode 100644 index 000000000000..cadd7f616a9d --- /dev/null +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreService.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.api.gax.grpc.testing.MockGrpcService; +import com.google.protobuf.AbstractMessage; +import io.grpc.ServerServiceDefinition; +import java.util.ArrayList; +import java.util.List; + +/** Mock the BigLake Metastore service for testing. */ +public class MockMetastoreService implements MockGrpcService { + + private final MockMetastoreServiceImpl serviceImpl; + + public MockMetastoreService() { + serviceImpl = new MockMetastoreServiceImpl(); + } + + @Override + public List getRequests() { + return new ArrayList(); + } + + @Override + public void addResponse(AbstractMessage response) {} + + @Override + public void addException(Exception exception) {} + + @Override + public ServerServiceDefinition getServiceDefinition() { + return serviceImpl.bindService(); + } + + @Override + public void reset() { + serviceImpl.reset(); + } +} diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreServiceImpl.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreServiceImpl.java new file mode 100644 index 000000000000..84de02724e08 --- /dev/null +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/MockMetastoreServiceImpl.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.biglake; + +import com.google.cloud.bigquery.biglake.v1.Catalog; +import com.google.cloud.bigquery.biglake.v1.CreateCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.CreateDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.CreateTableRequest; +import com.google.cloud.bigquery.biglake.v1.Database; +import com.google.cloud.bigquery.biglake.v1.DeleteCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.DeleteDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.DeleteTableRequest; +import com.google.cloud.bigquery.biglake.v1.GetCatalogRequest; +import com.google.cloud.bigquery.biglake.v1.GetDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.GetTableRequest; +import com.google.cloud.bigquery.biglake.v1.ListCatalogsRequest; +import com.google.cloud.bigquery.biglake.v1.ListCatalogsResponse; +import com.google.cloud.bigquery.biglake.v1.ListDatabasesRequest; +import com.google.cloud.bigquery.biglake.v1.ListDatabasesResponse; +import com.google.cloud.bigquery.biglake.v1.ListTablesRequest; +import com.google.cloud.bigquery.biglake.v1.ListTablesResponse; +import com.google.cloud.bigquery.biglake.v1.MetastoreServiceGrpc.MetastoreServiceImplBase; +import com.google.cloud.bigquery.biglake.v1.RenameTableRequest; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.UpdateDatabaseRequest; +import com.google.cloud.bigquery.biglake.v1.UpdateTableRequest; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; + +/** Mock the BigLake Metastore service for testing. */ +public class MockMetastoreServiceImpl extends MetastoreServiceImplBase { + + private final Map catalogs; + private final Map dbs; + private final Map tables; + + public MockMetastoreServiceImpl() { + this.catalogs = new HashMap(); + this.dbs = new HashMap(); + this.tables = new HashMap(); + } + + public void reset() { + this.catalogs.clear(); + this.dbs.clear(); + this.tables.clear(); + } + + @Override + public void createCatalog( + CreateCatalogRequest request, StreamObserver responseObserver) { + String name = String.format("%s/catalogs/%s", request.getParent(), request.getCatalogId()); + if (catalogs.containsKey(name)) { + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription(String.format("Catalog already exists: %s", request.getCatalogId())) + .asRuntimeException()); + return; + } + + Catalog catalog = request.getCatalog().toBuilder().setName(name).build(); + catalogs.put(name, catalog); + responseObserver.onNext(catalog); + responseObserver.onCompleted(); + } + + @Override + public void deleteCatalog( + DeleteCatalogRequest request, StreamObserver responseObserver) { + String name = request.getName(); + if (catalogs.containsKey(name)) { + responseObserver.onNext(catalogs.remove(name)); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Catalog %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void getCatalog(GetCatalogRequest request, StreamObserver responseObserver) { + String name = request.getName(); + if (catalogs.containsKey(name)) { + responseObserver.onNext(catalogs.get(name)); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Catalog %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void listCatalogs( + ListCatalogsRequest request, StreamObserver responseObserver) { + List result = + catalogs.values().stream() + .filter(c -> c.getName().startsWith(request.getParent())) + .collect(ImmutableList.toImmutableList()); + responseObserver.onNext(ListCatalogsResponse.newBuilder().addAllCatalogs(result).build()); + responseObserver.onCompleted(); + } + + @Override + public void createDatabase( + CreateDatabaseRequest request, StreamObserver responseObserver) { + if (request.getDatabaseId().contains("/")) { + responseObserver.onError( + Status.INVALID_ARGUMENT.withDescription("Database ID is invalid").asRuntimeException()); + return; + } + + String name = String.format("%s/databases/%s", request.getParent(), request.getDatabaseId()); + if (dbs.containsKey(name)) { + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription( + String.format("Database already exists: %s", request.getDatabaseId())) + .asRuntimeException()); + return; + } + + Database db = request.getDatabase().toBuilder().setName(name).build(); + dbs.put(name, db); + responseObserver.onNext(db); + responseObserver.onCompleted(); + } + + @Override + public void deleteDatabase( + DeleteDatabaseRequest request, StreamObserver responseObserver) { + String name = request.getName(); + if (dbs.containsKey(name)) { + responseObserver.onNext(dbs.remove(name)); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Database %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void updateDatabase( + UpdateDatabaseRequest request, StreamObserver responseObserver) { + String name = request.getDatabase().getName(); + if (dbs.containsKey(name)) { + Database.Builder dbBuilder = dbs.get(name).toBuilder(); + dbBuilder + .getHiveOptionsBuilder() + .clearParameters() + .putAllParameters(request.getDatabase().getHiveOptions().getParameters()); + Database newDb = dbBuilder.build(); + dbs.put(name, newDb); + responseObserver.onNext(newDb); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Database %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void getDatabase(GetDatabaseRequest request, StreamObserver responseObserver) { + String name = request.getName(); + if (dbs.containsKey(name)) { + responseObserver.onNext(dbs.get(name)); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Database %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void listDatabases( + ListDatabasesRequest request, StreamObserver responseObserver) { + List result = + dbs.values().stream() + .filter(db -> db.getName().startsWith(request.getParent())) + .collect(ImmutableList.toImmutableList()); + responseObserver.onNext(ListDatabasesResponse.newBuilder().addAllDatabases(result).build()); + responseObserver.onCompleted(); + } + + @Override + public void createTable(CreateTableRequest request, StreamObserver
responseObserver) { + String name = String.format("%s/tables/%s", request.getParent(), request.getTableId()); + if (tables.containsKey(name)) { + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription(String.format("Table already exists: %s", request.getTableId())) + .asRuntimeException()); + return; + } + + Table table = + request.getTable().toBuilder().setName(name).setEtag(UUID.randomUUID().toString()).build(); + tables.put(name, table); + responseObserver.onNext(table); + responseObserver.onCompleted(); + } + + @Override + public void deleteTable(DeleteTableRequest request, StreamObserver
responseObserver) { + String name = request.getName(); + if (tables.containsKey(name)) { + responseObserver.onNext(tables.remove(name)); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Table %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void updateTable(UpdateTableRequest request, StreamObserver
responseObserver) { + String name = request.getTable().getName(); + if (tables.containsKey(name)) { + Table.Builder tableBuilder = tables.get(name).toBuilder(); + tableBuilder + .getHiveOptionsBuilder() + .clearParameters() + .putAllParameters(request.getTable().getHiveOptions().getParameters()); + Table newTable = tableBuilder.setEtag(UUID.randomUUID().toString()).build(); + tables.put(name, newTable); + responseObserver.onNext(newTable); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Table %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void renameTable(RenameTableRequest request, StreamObserver
responseObserver) { + if (!tables.containsKey(request.getName())) { + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription("Table does not exist or permission denied") + .asRuntimeException()); + return; + } + + if (tables.containsKey(request.getNewName())) { + responseObserver.onError( + Status.ALREADY_EXISTS + .withDescription("Table does not exist or permission denied") + .asRuntimeException()); + return; + } + + Table table = tables.get(request.getName()); + Table newTable = + table + .toBuilder() + .setName(request.getNewName()) + .setEtag(UUID.randomUUID().toString()) + .build(); + tables.put(request.getNewName(), newTable); + tables.remove(request.getName()); + responseObserver.onNext(newTable); + responseObserver.onCompleted(); + } + + @Override + public void getTable(GetTableRequest request, StreamObserver
responseObserver) { + String name = request.getName(); + if (tables.containsKey(name)) { + responseObserver.onNext(tables.get(name)); + responseObserver.onCompleted(); + return; + } + + responseObserver.onError( + Status.PERMISSION_DENIED + .withDescription(String.format("Table %s does not exist or permission denied", name)) + .asRuntimeException()); + } + + @Override + public void listTables( + ListTablesRequest request, StreamObserver responseObserver) { + List
result = + tables.values().stream() + .filter(t -> t.getName().startsWith(request.getParent())) + .collect(ImmutableList.toImmutableList()); + responseObserver.onNext(ListTablesResponse.newBuilder().addAllTables(result).build()); + responseObserver.onCompleted(); + } +} diff --git a/versions.props b/versions.props index 232d32bd8574..a01e4db4461e 100644 --- a/versions.props +++ b/versions.props @@ -30,7 +30,7 @@ com.emc.ecs:object-client-bundle = 3.3.2 org.immutables:value = 2.9.2 net.snowflake:snowflake-jdbc = 3.13.30 io.delta:delta-standalone_* = 0.6.0 -com.google.cloud:google-cloud-biglake = 0.3.0 +com.google.cloud:google-cloud-biglake = 0.6.0 # test deps org.junit.vintage:junit-vintage-engine = 5.9.2 @@ -50,3 +50,4 @@ org.eclipse.jetty:* = 9.4.43.v20210629 org.testcontainers:* = 1.17.6 io.delta:delta-core_* = 2.2.0 org.awaitility:awaitility = 4.2.0 +com.google.api.grpc:grpc-google-cloud-biglake-v1 = 0.6.0 From 836d8f570a65b5800fe5f79a91c0a436df58c14f Mon Sep 17 00:00:00 2001 From: coufon Date: Wed, 7 Jun 2023 16:11:42 +0000 Subject: [PATCH 10/22] runs a grpc server per test to avoid interference --- .../gcp/biglake/BigLakeCatalogTest.java | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 11ecb009c999..1d3e3a6a3a95 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -54,8 +54,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.junit.Rule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -71,34 +70,25 @@ public class BigLakeCatalogTest extends CatalogTests { private static final String GCP_REGION = "us"; private static final String CATALOG_ID = "biglake"; - private static MockMetastoreService mockMetastoreService; - private static MockServiceHelper mockServiceHelper; + private String warehouseLocation; + // For tests using a BigLake catalog connecting to a mocked service. + private MockMetastoreService mockMetastoreService; + private MockServiceHelper mockServiceHelper; private LocalChannelProvider channelProvider; private BigLakeCatalog bigLakeCatalogUsingMockService; - private String warehouseLocation; - + // For tests using a BigLake catalog with a mocked client. private BigLakeClient mockBigLakeClient; private BigLakeCatalog bigLakeCatalogUsingMockClient; - @BeforeAll - public static void setUpStaticBigLakeService() throws Exception { + @BeforeEach + public void setUp() throws Exception { mockMetastoreService = new MockMetastoreService(); mockServiceHelper = new MockServiceHelper( UUID.randomUUID().toString(), Arrays.asList(mockMetastoreService)); mockServiceHelper.start(); - } - - @AfterAll - public static void stopStaticBigLakeService() { - mockServiceHelper.stop(); - } - - @BeforeEach - public void createCatalog() throws Exception { - mockMetastoreService.reset(); File warehouse = temp.toFile(); warehouseLocation = warehouse.getAbsolutePath(); @@ -133,6 +123,11 @@ public void createCatalog() throws Exception { CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, mockBigLakeClient); } + @AfterEach + public void tearDown() { + mockServiceHelper.stop(); + } + @Override protected boolean requiresNamespaceCreate() { return true; From 592eb6331f52813bf7457b20665d25f19e3f18df Mon Sep 17 00:00:00 2001 From: coufon Date: Fri, 9 Jun 2023 20:24:03 +0000 Subject: [PATCH 11/22] fix :iceberg-gcp:checkstyleMain failure --- .../org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 1d3e3a6a3a95..120dfff9107d 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -83,7 +83,7 @@ public class BigLakeCatalogTest extends CatalogTests { private BigLakeCatalog bigLakeCatalogUsingMockClient; @BeforeEach - public void setUp() throws Exception { + public void before() throws Exception { mockMetastoreService = new MockMetastoreService(); mockServiceHelper = new MockServiceHelper( @@ -124,7 +124,7 @@ public void setUp() throws Exception { } @AfterEach - public void tearDown() { + public void after() { mockServiceHelper.stop(); } From c875ed39605cc06753d5bb237b678859d06bbe9c Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 11 Jul 2023 23:34:48 +0000 Subject: [PATCH 12/22] fix style issues from review --- .../org/apache/iceberg/gcp/GCPProperties.java | 14 ++ .../iceberg/gcp/biglake/BigLakeCatalog.java | 226 ++++++++---------- .../iceberg/gcp/biglake/BigLakeClient.java | 14 +- .../gcp/biglake/BigLakeTableOperations.java | 16 +- .../gcp/biglake/BigLakeCatalogTest.java | 95 +++----- .../biglake/BigLakeTableOperationsTest.java | 3 +- 6 files changed, 161 insertions(+), 207 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java b/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java index 457a76313d15..700571f35c2a 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java @@ -23,6 +23,7 @@ import java.util.Optional; public class GCPProperties implements Serializable { + // For Google Cloud Storage (GCS). // Service Options public static final String GCS_PROJECT_ID = "gcs.project-id"; public static final String GCS_CLIENT_LIB_TOKEN = "gcs.client-lib-token"; @@ -36,6 +37,19 @@ public class GCPProperties implements Serializable { public static final String GCS_CHANNEL_READ_CHUNK_SIZE = "gcs.channel.read.chunk-size-bytes"; public static final String GCS_CHANNEL_WRITE_CHUNK_SIZE = "gcs.channel.write.chunk-size-bytes"; + // For BigQuery BigLake Metastore. + // The endpoint of BigLake API. + // Optional, default to BigLakeCatalog.DEFAULT_BIGLAKE_SERVICE_ENDPOINT. + public static final String BIGLAKE_ENDPOINT = "biglake.endpoint"; + // The GCP project ID. Required. + public static final String BIGLAKE_PROJECT_ID = "biglake.project-id"; + // The GCP region (https://cloud.google.com/bigquery/docs/locations). Required. + public static final String BIGLAKE_GCP_REGION = "biglake.region"; + // The BigLake Metastore catalog ID. It is the container resource of databases and tables. + // It links a BLMS catalog with this Iceberg catalog. + // Optional, default to the Spark catalog plugin name. + public static final String BIGLAKE_CATALOG_ID = "biglake.catalog-id"; + private String projectId; private String clientLibToken; private String serviceHost; diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index f0f344b22252..bde1a6241718 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -26,7 +26,6 @@ import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -42,6 +41,7 @@ import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.hadoop.Configurable; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.ResolvingFileIO; @@ -51,7 +51,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.util.LocationUtil; import org.slf4j.Logger; @@ -61,31 +60,17 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog implements SupportsNamespaces, Configurable { - // TODO: to move the configs to GCPProperties.java. - // User provided properties. - // The endpoint of BigLake API. Optional, default to DEFAULT_BIGLAKE_SERVICE_ENDPOINT. - public static final String PROPERTIES_KEY_BIGLAKE_ENDPOINT = "biglake.endpoint"; - // The GCP project ID. Required. - public static final String PROPERTIES_KEY_GCP_PROJECT = "biglake.project-id"; - // The GCP location (https://cloud.google.com/bigquery/docs/locations). Optional, default to - // DEFAULT_GCP_LOCATION. - public static final String PROPERTIES_KEY_GCP_LOCATION = "biglake.location"; - // The BLMS catalog ID. It is the container resource of databases and tables. - // It links a BLMS catalog with this Iceberg catalog. - public static final String PROPERTIES_KEY_BLMS_CATALOG = "biglake.catalog"; - public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; - public static final String DEFAULT_GCP_LOCATION = "us"; private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); // The name of this Iceberg catalog plugin: spark.sql.catalog.. private String catalogPulginName; - private Map catalogProperties; - private FileIO fileIO; + private Map properties; + private FileIO io; private Object conf; private String projectId; - private String location; + private String region; // BLMS catalog ID and fully qualified name. private String catalogId; private CatalogName catalogName; @@ -99,76 +84,80 @@ public BigLakeCatalog() {} @Override public void initialize(String inputName, Map properties) { Preconditions.checkArgument( - properties.containsKey(PROPERTIES_KEY_GCP_PROJECT), "GCP project must be specified"); - String propProjectId = properties.get(PROPERTIES_KEY_GCP_PROJECT); - String propLocation = - properties.getOrDefault(PROPERTIES_KEY_GCP_LOCATION, DEFAULT_GCP_LOCATION); + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), + "GCP project ID must be specified"); + String projectId = properties.get(GCPProperties.BIGLAKE_PROJECT_ID); + + Preconditions.checkArgument( + properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), "GCP region must be specified"); + String region = properties.get(GCPProperties.BIGLAKE_GCP_REGION); - BigLakeClient newClient; + BigLakeClient client; try { // TODO: to add more auth options of the client. Currently it uses default auth // (https://github.com/googleapis/google-cloud-java#application-default-credentials) // that works on GCP services (e.g., GCE, GKE, Dataproc). - newClient = + client = new BigLakeClient( properties.getOrDefault( - PROPERTIES_KEY_BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), - propProjectId, - propLocation); + GCPProperties.BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), + projectId, + region); } catch (IOException e) { throw new ServiceFailureException(e, "Creating BigLake client failed"); } - initialize(inputName, properties, propProjectId, propLocation, newClient); + + initialize(inputName, properties, projectId, region, client); } @VisibleForTesting void initialize( String inputName, Map properties, - String propProjectId, - String propLocation, - BigLakeClient bigLakeClient) { + String projectId, + String region, + BigLakeClient client) { this.catalogPulginName = inputName; - this.catalogProperties = ImmutableMap.copyOf(properties); - this.projectId = propProjectId; - this.location = propLocation; - Preconditions.checkNotNull(bigLakeClient, "BigLake client must not be null"); - this.client = bigLakeClient; - - // Users can specify the BigLake catalog ID, otherwise catalog plugin will be used. - this.catalogId = properties.getOrDefault(PROPERTIES_KEY_BLMS_CATALOG, inputName); - this.catalogName = CatalogName.of(projectId, location, catalogId); - LOG.info("Use BigLake catalog: {}", catalogName.toString()); - - String fileIOImpl = - properties.getOrDefault(CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); - this.fileIO = CatalogUtil.loadFileIO(fileIOImpl, properties, conf); + this.properties = ImmutableMap.copyOf(properties); + this.projectId = projectId; + this.region = region; + Preconditions.checkNotNull(client, "BigLake client must not be null"); + this.client = client; + + // Users can specify the BigLake catalog ID, otherwise catalog plugin name will be used. + // For example, "spark.sql.catalog.=org.apache.iceberg.spark.SparkCatalog" + // specifies the plugin name "". + this.catalogId = this.properties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, inputName); + this.catalogName = CatalogName.of(projectId, region, catalogId); + + String ioImpl = + this.properties.getOrDefault( + CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); + this.io = CatalogUtil.loadFileIO(ioImpl, this.properties, conf); } @Override protected TableOperations newTableOps(TableIdentifier identifier) { // The identifier of metadata tables is like "ns.table.files". - // We return a non-existing table in this case (empty table ID is disallowed in BigLake + // Return a non-existing table in this case (empty table ID is disallowed in BigLake // Metastore), loadTable will try loadMetadataTable. if (identifier.namespace().levels().length > 1 && MetadataTableType.from(identifier.name()) != null) { return new BigLakeTableOperations( - client, fileIO, getTableName(identifier.namespace().level(0), /* tableId= */ "")); + client, io, tableName(identifier.namespace().level(0), /* tableId= */ "")); } return new BigLakeTableOperations( - client, - fileIO, - getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); + client, io, tableName(databaseId(identifier.namespace()), identifier.name())); } @Override protected String defaultWarehouseLocation(TableIdentifier identifier) { - String locationUri = getDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); + String locationUri = loadDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); return String.format( "%s/%s", Strings.isNullOrEmpty(locationUri) - ? getDatabaseLocation(getDatabaseId(identifier.namespace())) + ? databaseLocation(databaseId(identifier.namespace())) : locationUri, identifier.name()); } @@ -185,8 +174,8 @@ public List listTables(Namespace namespace) { : ImmutableList.of(TableIdentifier.of("placeholder")); } - return Streams.stream(client.listTables(getDatabaseName(namespace))) - .map(BigLakeCatalog::getTableIdentifier) + return Streams.stream(client.listTables(databaseName(namespace))) + .map(BigLakeCatalog::tableIdentifier) .collect(ImmutableList.toImmutableList()); } @@ -196,10 +185,8 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. TableMetadata lastMetadata = ops.current(); try { - client.deleteTable( - getTableName(getDatabaseId(identifier.namespace()), /* tableId= */ identifier.name())); + client.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); } catch (NoSuchTableException e) { - LOG.warn("Dropping table failed", e); return false; } @@ -212,12 +199,15 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { @Override public void renameTable(TableIdentifier from, TableIdentifier to) { - String fromDbId = getDatabaseId(from.namespace()); - String toDbId = getDatabaseId(to.namespace()); + String fromDbId = databaseId(from.namespace()); + String toDbId = databaseId(to.namespace()); Preconditions.checkArgument( - fromDbId.equals(toDbId), "New table name must be in the same database"); - client.renameTable(getTableName(fromDbId, from.name()), getTableName(toDbId, to.name())); + fromDbId.equals(toDbId), + "Cannot rename table %s to %s: database must match", + from.toString(), + to.toString()); + client.renameTable(tableName(fromDbId, from.name()), tableName(toDbId, to.name())); } @Override @@ -228,16 +218,17 @@ public void createNamespace(Namespace namespace, Map metadata) { LOG.info("Created BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { // Create a database. - String dbId = namespace.level(0); + String dbId = databaseId(namespace); Database.Builder builder = Database.newBuilder().setType(Database.Type.HIVE); builder .getHiveOptionsBuilder() .putAllParameters(metadata) - .setLocationUri(getDatabaseLocation(dbId)); + .setLocationUri(databaseLocation(dbId)); - client.createDatabase(DatabaseName.of(projectId, location, catalogId, dbId), builder.build()); + client.createDatabase(DatabaseName.of(projectId, region, catalogId, dbId), builder.build()); } else { - throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + throw new IllegalArgumentException( + String.format("Invalid namespace (too long): %s", namespace)); } } @@ -251,7 +242,7 @@ public List listNamespaces(Namespace namespace) { } return Streams.stream(client.listDatabases(catalogName)) - .map(BigLakeCatalog::getNamespace) + .map(BigLakeCatalog::namespace) .collect(ImmutableList.toImmutableList()); } @@ -263,14 +254,14 @@ public boolean dropNamespace(Namespace namespace) { client.deleteCatalog(catalogName); LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { - client.deleteDatabase(getDatabaseName(namespace)); - // We don't delete the data file folder for safety. It aligns with HMS's default behavior. - // We can support database or catalog level config controlling file deletion in future. + client.deleteDatabase(databaseName(namespace)); + // Don't delete the data file folder for safety. It aligns with HMS's default behavior. + // To support database or catalog level config controlling file deletion in future. } else { - throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + LOG.warn("Invalid namespace (too long): %s", namespace); + return false; } } catch (NoSuchNamespaceException e) { - LOG.warn("Dropping namespace failed", e); return false; } @@ -279,40 +270,19 @@ public boolean dropNamespace(Namespace namespace) { @Override public boolean setProperties(Namespace namespace, Map properties) { - Database.Builder builder; - try { - builder = getDatabase(namespace).toBuilder(); - } catch (IllegalArgumentException e) { - LOG.warn( - "setProperties is only supported for tables and databases, namespace {} is not supported", - namespace.levels().length == 0 ? "empty" : namespace.toString(), - e); - return false; - } - - HiveDatabaseOptions.Builder optionsBuilder = builder.getHiveOptionsBuilder(); + HiveDatabaseOptions.Builder optionsBuilder = + loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); properties.forEach(optionsBuilder::putParameters); - client.updateDatabaseParameters(getDatabaseName(namespace), optionsBuilder.getParametersMap()); + client.updateDatabaseParameters(databaseName(namespace), optionsBuilder.getParametersMap()); return true; } @Override public boolean removeProperties(Namespace namespace, Set properties) { - Database.Builder builder; - try { - builder = getDatabase(namespace).toBuilder(); - } catch (IllegalArgumentException e) { - LOG.warn( - "removeProperties is only supported for tables and databases, namespace {} is not" - + " supported", - namespace.levels().length == 0 ? "empty" : namespace.toString(), - e); - return false; - } - - HiveDatabaseOptions.Builder optionsBuilder = builder.getHiveOptionsBuilder(); + HiveDatabaseOptions.Builder optionsBuilder = + loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); properties.forEach(optionsBuilder::removeParameters); - client.updateDatabaseParameters(getDatabaseName(namespace), optionsBuilder.getParametersMap()); + client.updateDatabaseParameters(databaseName(namespace), optionsBuilder.getParametersMap()); return true; } @@ -321,11 +291,11 @@ public Map loadNamespaceMetadata(Namespace namespace) { if (namespace.levels().length == 0) { // Calls getCatalog to check existence. BLMS catalog has no metadata today. client.getCatalog(catalogName); - return new HashMap(); + return ImmutableMap.of(); } else if (namespace.levels().length == 1) { - return getMetadata(getDatabase(namespace)); + return metadata(loadDatabase(namespace)); } else { - throw new IllegalArgumentException(invalidNamespaceMessage(namespace)); + throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } } @@ -336,7 +306,7 @@ public String name() { @Override protected Map properties() { - return catalogProperties == null ? ImmutableMap.of() : catalogProperties; + return properties == null ? ImmutableMap.of() : properties; } @Override @@ -344,55 +314,49 @@ public void setConf(Object conf) { this.conf = conf; } - private String getDatabaseLocation(String dbId) { + private String databaseLocation(String dbId) { String warehouseLocation = - LocationUtil.stripTrailingSlash( - catalogProperties.get(CatalogProperties.WAREHOUSE_LOCATION)); + LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set"); return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); } - private static TableIdentifier getTableIdentifier(Table table) { + private static TableIdentifier tableIdentifier(Table table) { TableName tableName = TableName.parse(table.getName()); return TableIdentifier.of(Namespace.of(tableName.getDatabase()), tableName.getTable()); } - private static Namespace getNamespace(Database db) { + private static Namespace namespace(Database db) { return Namespace.of(DatabaseName.parse(db.getName()).getDatabase()); } - private TableName getTableName(String dbId, String tableId) { - return TableName.of(projectId, location, catalogId, dbId, tableId); + private TableName tableName(String dbId, String tableId) { + return TableName.of(projectId, region, catalogId, dbId, tableId); } - private String getDatabaseId(Namespace namespace) { - Preconditions.checkArgument( - namespace.levels().length == 1, - "BigLake database namespace must use format ., invalid namespace: %s", - namespace); + private String databaseId(Namespace namespace) { + if (namespace.levels().length != 1) { + throw new NoSuchNamespaceException( + "BigLake database namespace must use format ., invalid namespace: %s", + namespace); + } + return namespace.level(0); } - private DatabaseName getDatabaseName(Namespace namespace) { - return DatabaseName.of(projectId, location, catalogId, getDatabaseId(namespace)); + private DatabaseName databaseName(Namespace namespace) { + return DatabaseName.of(projectId, region, catalogId, databaseId(namespace)); } - private Database getDatabase(Namespace namespace) { - return client.getDatabase(getDatabaseName(namespace)); + private Database loadDatabase(Namespace namespace) { + return client.getDatabase(databaseName(namespace)); } - private static Map getMetadata(Database db) { + private static Map metadata(Database db) { HiveDatabaseOptions options = db.getHiveOptions(); - Map result = Maps.newHashMap(); - result.putAll(options.getParameters()); - result.put("location", options.getLocationUri()); - return result; - } - - private static String invalidNamespaceMessage(Namespace namespace) { - return String.format( - "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" - + " namespace: %s", - namespace); + return new ImmutableMap.Builder() + .putAll(options.getParameters()) + .put("location", options.getLocationUri()) + .build(); } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index a6fb344e4025..de20f9f77d55 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -56,7 +56,7 @@ final class BigLakeClient { private final String projectId; - private final String location; + private final String region; private final MetastoreServiceClient stub; /** @@ -64,12 +64,12 @@ final class BigLakeClient { * * @param settings BigLake service settings * @param projectId GCP project ID - * @param location GCP region supported by BigLake, e.g., "us" + * @param region GCP region supported by BigLake, e.g., "us" */ - BigLakeClient(MetastoreServiceSettings settings, String projectId, String location) + BigLakeClient(MetastoreServiceSettings settings, String projectId, String region) throws IOException { this.projectId = projectId; - this.location = location; + this.region = region; this.stub = MetastoreServiceClient.create(settings); } @@ -78,13 +78,13 @@ final class BigLakeClient { * * @param biglakeEndpoint BigLake service gRPC endpoint, e.g., "biglake.googleapis.com:443" * @param projectId GCP project ID - * @param location GCP region supported by BigLake, e.g., "us" + * @param region GCP region supported by BigLake, e.g., "us" */ - BigLakeClient(String biglakeEndpoint, String projectId, String location) throws IOException { + BigLakeClient(String biglakeEndpoint, String projectId, String region) throws IOException { this( MetastoreServiceSettings.newBuilder().setEndpoint(biglakeEndpoint).build(), projectId, - location); + region); } public Catalog createCatalog(CatalogName name, Catalog catalog) { diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 56ec4f8d7e88..77da15da522b 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -35,7 +35,7 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,12 +45,12 @@ public final class BigLakeTableOperations extends BaseMetastoreTableOperations { private static final Logger LOG = LoggerFactory.getLogger(BigLakeTableOperations.class); private final BigLakeClient client; - private final FileIO fileIO; + private final FileIO io; private final TableName tableName; - BigLakeTableOperations(BigLakeClient client, FileIO fileIO, TableName tableName) { + BigLakeTableOperations(BigLakeClient client, FileIO io, TableName tableName) { this.client = client; - this.fileIO = fileIO; + this.io = io; this.tableName = tableName; } @@ -63,7 +63,7 @@ public void doRefresh() { HiveTableOptions hiveOptions = client.getTable(tableName).getHiveOptions(); if (!hiveOptions.containsParameters(METADATA_LOCATION_PROP)) { throw new NoSuchIcebergTableException( - "Table %s is not a valid Iceberg table, metadata location not found", tableName()); + "Invalid Iceberg table %s: missing metadata location", tableName()); } metadataLocation = hiveOptions.getParametersOrThrow(METADATA_LOCATION_PROP); @@ -128,7 +128,7 @@ public String tableName() { @Override public FileIO io() { - return fileIO; + return io; } private void createTable(String newMetadataLocation, TableMetadata metadata) { @@ -203,7 +203,7 @@ private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) // Follow Iceberg's HiveTableOperations to populate more table parameters for HMS compatibility. private Map buildTableParameters( String metadataFileLocation, TableMetadata metadata) { - Map parameters = Maps.newHashMap(); + ImmutableMap.Builder parameters = new ImmutableMap.Builder(); parameters.putAll(metadata.properties()); if (metadata.uuid() != null) { parameters.put(TableProperties.UUID, metadata.uuid()); @@ -217,6 +217,6 @@ private Map buildTableParameters( // Follow HMS to use the EXTERNAL type. parameters.put("EXTERNAL", "TRUE"); parameters.put("table_type", "ICEBERG"); - return parameters; + return parameters.build(); } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 120dfff9107d..243e7b332541 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -32,8 +32,6 @@ import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; -import com.google.api.gax.rpc.InvalidArgumentException; -import com.google.api.pathtemplate.ValidationException; import com.google.cloud.bigquery.biglake.v1.Catalog; import com.google.cloud.bigquery.biglake.v1.CatalogName; import com.google.cloud.bigquery.biglake.v1.Database; @@ -50,6 +48,8 @@ import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; @@ -95,7 +95,7 @@ public void before() throws Exception { ImmutableMap properties = ImmutableMap.of( - BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, + GCPProperties.BIGLAKE_PROJECT_ID, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); @@ -143,28 +143,6 @@ protected boolean supportsNamesWithSlashes() { return false; } - @Test - public void testNamespaceWithSlash() { - BigLakeCatalog catalog = catalog(); - - Exception exception = - assertThrows( - InvalidArgumentException.class, () -> catalog.createNamespace(Namespace.of("new/db"))); - assertTrue(exception.getMessage().contains("Database ID is invalid")); - } - - @Test - public void testTableNameWithSlash() { - BigLakeCatalog catalog = catalog(); - - catalog.createNamespace(Namespace.of("ns")); - TableIdentifier ident = TableIdentifier.of("ns", "tab/le"); - - Exception exception = - assertThrows(ValidationException.class, () -> catalog.buildTable(ident, SCHEMA).create()); - assertEquals("Invalid character \"/\" in path section \"tab/le\".", exception.getMessage()); - } - @Test public void testDefaultWarehouseWithDatabaseLocation_asExpected() { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) @@ -197,7 +175,8 @@ public void testRenameTable_differentDatabase_fail() { () -> bigLakeCatalogUsingMockClient.renameTable( TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); - assertEquals("New table name must be in the same database", exception.getMessage()); + assertEquals( + "Cannot rename table db0.t1 to db1.t2: database must match", exception.getMessage()); } @Test @@ -216,10 +195,7 @@ public void testCreateNamespace_failWhenInvalid() throws Exception { () -> bigLakeCatalogUsingMockClient.createNamespace( Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())); - assertEquals( - "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" - + " namespace: n0.n1", - exception.getMessage()); + assertEquals("Invalid namespace (too long): n0.n1", exception.getMessage()); } @Test @@ -258,26 +234,23 @@ public void testListTables_emptyNamespace_checkCatalogEmptiness() { @Test public void testDropNamespace_failWhenInvalid() throws Exception { - Exception exception = - assertThrows( - IllegalArgumentException.class, - () -> - bigLakeCatalogUsingMockClient.dropNamespace( - Namespace.of(new String[] {"n0", "n1"}))); - assertEquals( - "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" - + " namespace: n0.n1", - exception.getMessage()); + assertFalse( + bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))); } @Test public void testSetProperties_failWhenNamespacesAreInvalid() throws Exception { - assertFalse( - bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {}), ImmutableMap.of())); - assertFalse( - bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); + assertThrows( + NoSuchNamespaceException.class, + () -> + bigLakeCatalogUsingMockClient.setProperties( + Namespace.of(new String[] {}), ImmutableMap.of())); + + assertThrows( + NoSuchNamespaceException.class, + () -> + bigLakeCatalogUsingMockClient.setProperties( + Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); } @Test @@ -303,12 +276,17 @@ public void testSetProperties_succeedForDatabase() throws Exception { @Test public void testRemoveProperties_failWhenNamespacesAreInvalid() throws Exception { - assertFalse( - bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {}), ImmutableSet.of())); - assertFalse( - bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())); + assertThrows( + NoSuchNamespaceException.class, + () -> + bigLakeCatalogUsingMockClient.removeProperties( + Namespace.of(new String[] {}), ImmutableSet.of())); + + assertThrows( + NoSuchNamespaceException.class, + () -> + bigLakeCatalogUsingMockClient.removeProperties( + Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())); } @Test @@ -362,14 +340,11 @@ public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { public void testLoadNamespaceMetadata_failWhenInvalid() throws Exception { Exception exception = assertThrows( - IllegalArgumentException.class, + NoSuchNamespaceException.class, () -> bigLakeCatalogUsingMockClient.loadNamespaceMetadata( Namespace.of(new String[] {"n0", "n1"}))); - assertEquals( - "BigLake catalog namespace can have zero (catalog) or one level (database), invalid" - + " namespace: n0.n1", - exception.getMessage()); + assertEquals("Namespace does not exist: n0.n1", exception.getMessage()); } @Test @@ -378,11 +353,11 @@ public void testSetBigLakeCatalogInProperties_asExpected() throws Exception { catalog.initialize( CATALOG_ID, /* properties= */ ImmutableMap.of( - BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, + GCPProperties.BIGLAKE_PROJECT_ID, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation, - BigLakeCatalog.PROPERTIES_KEY_BLMS_CATALOG, + GCPProperties.BIGLAKE_CATALOG_ID, "customized_catalog"), GCP_PROJECT, GCP_REGION, @@ -399,7 +374,7 @@ public void testSetBigLakeCatalogInProperties_asExpected() throws Exception { public void testNewTableOps_failedForInvalidNamespace() throws Exception { Exception exception = assertThrows( - IllegalArgumentException.class, + NoSuchNamespaceException.class, () -> bigLakeCatalogUsingMockClient.newTableOps( TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))); diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 88cb5573abc1..324bea4c9888 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -46,6 +46,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; import org.junit.Before; @@ -87,7 +88,7 @@ public void before() throws Exception { warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); ImmutableMap properties = ImmutableMap.of( - BigLakeCatalog.PROPERTIES_KEY_GCP_PROJECT, + GCPProperties.BIGLAKE_PROJECT_ID, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); From 425d9a8d91327887f155e70fd63d89ed1a142ae6 Mon Sep 17 00:00:00 2001 From: coufon Date: Wed, 12 Jul 2023 17:42:59 +0000 Subject: [PATCH 13/22] fix more review comments --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 62 +++++++++---------- .../gcp/biglake/BigLakeTableOperations.java | 8 ++- .../gcp/biglake/BigLakeCatalogTest.java | 30 +++++++-- .../biglake/BigLakeTableOperationsTest.java | 13 +++- 4 files changed, 73 insertions(+), 40 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index bde1a6241718..31723ad5e8a4 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -32,7 +32,6 @@ import org.apache.iceberg.BaseMetastoreCatalog; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.CatalogUtil; -import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.Namespace; @@ -50,7 +49,6 @@ import org.apache.iceberg.relocated.com.google.common.base.Strings; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.util.LocationUtil; import org.slf4j.Logger; @@ -65,7 +63,7 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); // The name of this Iceberg catalog plugin: spark.sql.catalog.. - private String catalogPulginName; + private String name; private Map properties; private FileIO io; private Object conf; @@ -89,7 +87,7 @@ public void initialize(String inputName, Map properties) { String projectId = properties.get(GCPProperties.BIGLAKE_PROJECT_ID); Preconditions.checkArgument( - properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), "GCP region must be specified"); + properties.containsKey(GCPProperties.BIGLAKE_GCP_REGION), "GCP region must be specified"); String region = properties.get(GCPProperties.BIGLAKE_GCP_REGION); BigLakeClient client; @@ -117,7 +115,7 @@ void initialize( String projectId, String region, BigLakeClient client) { - this.catalogPulginName = inputName; + this.name = inputName; this.properties = ImmutableMap.copyOf(properties); this.projectId = projectId; this.region = region; @@ -138,17 +136,8 @@ void initialize( @Override protected TableOperations newTableOps(TableIdentifier identifier) { - // The identifier of metadata tables is like "ns.table.files". - // Return a non-existing table in this case (empty table ID is disallowed in BigLake - // Metastore), loadTable will try loadMetadataTable. - if (identifier.namespace().levels().length > 1 - && MetadataTableType.from(identifier.name()) != null) { - return new BigLakeTableOperations( - client, io, tableName(identifier.namespace().level(0), /* tableId= */ "")); - } - return new BigLakeTableOperations( - client, io, tableName(databaseId(identifier.namespace()), identifier.name())); + client, io, name(), tableName(databaseId(identifier.namespace()), identifier.name())); } @Override @@ -164,19 +153,25 @@ protected String defaultWarehouseLocation(TableIdentifier identifier) { @Override public List listTables(Namespace namespace) { - // When deleting a BLMS catalog via `DROP NAMESPACE `, this method is called for - // verifying catalog emptiness. `namespace` is empty in this case, we list databases in - // this catalog instead. - // TODO: to return all tables in all databases in a BLMS catalog instead of a "placeholder". - if (namespace.levels().length == 0) { - return Iterables.isEmpty(client.listDatabases(catalogName)) - ? ImmutableList.of() - : ImmutableList.of(TableIdentifier.of("placeholder")); + ImmutableList dbNames; + if (namespace.isEmpty()) { + dbNames = + Streams.stream(client.listDatabases(catalogName)) + .map(db -> DatabaseName.parse(db.getName())) + .collect(ImmutableList.toImmutableList()); + } else { + dbNames = ImmutableList.of(databaseName(namespace)); } - return Streams.stream(client.listTables(databaseName(namespace))) - .map(BigLakeCatalog::tableIdentifier) - .collect(ImmutableList.toImmutableList()); + ImmutableList.Builder result = new ImmutableList.Builder(); + dbNames.stream() + .map( + dbName -> + Streams.stream(client.listTables(dbName)) + .map(BigLakeCatalog::tableIdentifier) + .collect(ImmutableList.toImmutableList())) + .forEach(result::addAll); + return result.build(); } @Override @@ -212,7 +207,7 @@ public void renameTable(TableIdentifier from, TableIdentifier to) { @Override public void createNamespace(Namespace namespace, Map metadata) { - if (namespace.levels().length == 0) { + if (namespace.isEmpty()) { // Used by `CREATE NAMESPACE `. Create a BLMS catalog linked with Iceberg catalog. client.createCatalog(catalogName, Catalog.getDefaultInstance()); LOG.info("Created BigLake catalog: {}", catalogName.toString()); @@ -234,7 +229,7 @@ public void createNamespace(Namespace namespace, Map metadata) { @Override public List listNamespaces(Namespace namespace) { - if (namespace.levels().length != 0) { + if (!namespace.isEmpty()) { // BLMS does not support namespaces under database or tables, returns empty. // It is called when dropping a namespace to make sure it's empty (listTables is called as // well), returns empty to unblock deletion. @@ -249,7 +244,7 @@ public List listNamespaces(Namespace namespace) { @Override public boolean dropNamespace(Namespace namespace) { try { - if (namespace.levels().length == 0) { + if (namespace.isEmpty()) { // Used by `DROP NAMESPACE `. Deletes the BLMS catalog linked by Iceberg catalog. client.deleteCatalog(catalogName); LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); @@ -288,7 +283,7 @@ public boolean removeProperties(Namespace namespace, Set properties) { @Override public Map loadNamespaceMetadata(Namespace namespace) { - if (namespace.levels().length == 0) { + if (namespace.isEmpty()) { // Calls getCatalog to check existence. BLMS catalog has no metadata today. client.getCatalog(catalogName); return ImmutableMap.of(); @@ -301,7 +296,7 @@ public Map loadNamespaceMetadata(Namespace namespace) { @Override public String name() { - return catalogPulginName; + return name; } @Override @@ -314,6 +309,11 @@ public void setConf(Object conf) { this.conf = conf; } + @Override + protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { + return tableIdentifier.namespace().levels().length == 1; + } + private String databaseLocation(String dbId) { String warehouseLocation = LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 77da15da522b..a6df46d83ea4 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -46,11 +46,14 @@ public final class BigLakeTableOperations extends BaseMetastoreTableOperations { private final BigLakeClient client; private final FileIO io; + // The catalog name. + private final String name; private final TableName tableName; - BigLakeTableOperations(BigLakeClient client, FileIO io, TableName tableName) { + BigLakeTableOperations(BigLakeClient client, FileIO io, String name, TableName tableName) { this.client = client; this.io = io; + this.name = name; this.tableName = tableName; } @@ -122,8 +125,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { @Override public String tableName() { - return String.format( - "%s.%s.%s", tableName.getCatalog(), tableName.getDatabase(), tableName.getTable()); + return String.format("%s.%s.%s", name, tableName.getDatabase(), tableName.getTable()); } @Override diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 243e7b332541..be8a3f03eacd 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -38,6 +38,8 @@ import com.google.cloud.bigquery.biglake.v1.DatabaseName; import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; +import com.google.cloud.bigquery.biglake.v1.Table; +import com.google.cloud.bigquery.biglake.v1.TableName; import java.io.File; import java.nio.file.Path; import java.util.Arrays; @@ -223,13 +225,33 @@ public void testListTables_emptyNamespace_noDatabase() { } @Test - public void testListTables_emptyNamespace_checkCatalogEmptiness() { + public void testListTables_emptyNamespace_listTablesInAllDbs() { + DatabaseName db1Name = DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db1"); + DatabaseName db2Name = DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db2"); + + TableName table1Name = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db1", "tbl1"); + TableName table2Name = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db1", "tbl2"); + TableName table3Name = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db2", "tbl3"); + when(mockBigLakeClient.listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID))) - .thenReturn(ImmutableList.of(Database.getDefaultInstance())); + .thenReturn( + ImmutableList.of( + Database.newBuilder().setName(db1Name.toString()).build(), + Database.newBuilder().setName(db2Name.toString()).build())); + + when(mockBigLakeClient.listTables(db1Name)) + .thenReturn( + ImmutableList.of( + Table.newBuilder().setName(table1Name.toString()).build(), + Table.newBuilder().setName(table2Name.toString()).build())); + when(mockBigLakeClient.listTables(db2Name)) + .thenReturn(ImmutableList.of(Table.newBuilder().setName(table3Name.toString()).build())); List result = bigLakeCatalogUsingMockClient.listTables(Namespace.of()); - assertEquals(1, result.size()); - assertEquals(TableIdentifier.of("placeholder"), result.get(0)); + assertEquals(3, result.size()); + assertEquals(TableIdentifier.of("db1", "tbl1"), result.get(0)); + assertEquals(TableIdentifier.of("db1", "tbl2"), result.get(1)); + assertEquals(TableIdentifier.of("db2", "tbl3"), result.get(2)); } @Test diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 324bea4c9888..9a293dff4727 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -63,6 +63,8 @@ public class BigLakeTableOperationsTest { @Rule public final MockitoRule mockito = MockitoJUnit.rule(); @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + private static final String CATALOG_NAME = "iceberg"; + private static final String GCP_PROJECT = "my-project"; private static final String GCP_REGION = "us"; private static final String CATALOG_ID = "biglake"; @@ -91,11 +93,13 @@ public void before() throws Exception { GCPProperties.BIGLAKE_PROJECT_ID, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, - warehouseLocation); + warehouseLocation, + GCPProperties.BIGLAKE_CATALOG_ID, + CATALOG_ID); bigLakeCatalog = new BigLakeCatalog(); bigLakeCatalog.setConf(new Configuration()); - bigLakeCatalog.initialize(CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, bigLakeClient); + bigLakeCatalog.initialize(CATALOG_NAME, properties, GCP_PROJECT, GCP_REGION, bigLakeClient); this.tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(SPARK_TABLE_ID); } @@ -153,6 +157,11 @@ public void testDoFresh_refreshReturnNullForNonIcebergTable() throws Exception { assertEquals(null, tableOps.refresh()); } + @Test + public void testTableName() throws Exception { + assertEquals(tableOps.tableName(), "iceberg.db.tbl"); + } + private Table createTestTable() throws IOException { TableIdentifier tableIdent = TableIdentifier.of(TABLE_NAME.getDatabase(), TABLE_NAME.getTable()); From 2a3878ff76597803327bd0fbed3bc60c449dd835 Mon Sep 17 00:00:00 2001 From: coufon Date: Thu, 13 Jul 2023 01:40:59 +0000 Subject: [PATCH 14/22] fix more comments on tests --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 86 ++++--- .../gcp/biglake/BigLakeTableOperations.java | 8 +- .../gcp/biglake/BigLakeCatalogTest.java | 219 +++++++----------- .../biglake/BigLakeTableOperationsTest.java | 58 ++--- 4 files changed, 176 insertions(+), 195 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 31723ad5e8a4..1abcbe8342fe 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -25,6 +25,7 @@ import com.google.cloud.bigquery.biglake.v1.HiveDatabaseOptions; import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; +import java.io.Closeable; import java.io.IOException; import java.util.List; import java.util.Map; @@ -42,6 +43,7 @@ import org.apache.iceberg.exceptions.ServiceFailureException; import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.hadoop.Configurable; +import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.ResolvingFileIO; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; @@ -56,7 +58,7 @@ /** Iceberg BigLake Metastore (BLMS) Catalog implementation. */ public final class BigLakeCatalog extends BaseMetastoreCatalog - implements SupportsNamespaces, Configurable { + implements Closeable, SupportsNamespaces, Configurable { public static final String DEFAULT_BIGLAKE_SERVICE_ENDPOINT = "biglake.googleapis.com:443"; @@ -64,15 +66,18 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog // The name of this Iceberg catalog plugin: spark.sql.catalog.. private String name; - private Map properties; + private Map bigLakeProperties; private FileIO io; private Object conf; - private String projectId; - private String region; + + private String bigLakeProjectId; + private String bigLakeRegion; // BLMS catalog ID and fully qualified name. private String catalogId; private CatalogName catalogName; - private BigLakeClient client; + private BigLakeClient bigLakeClient; + + private CloseableGroup closeableGroup; // Must have a no-arg constructor to be dynamically loaded // initialize(String name, Map properties) will be called to complete @@ -116,28 +121,36 @@ void initialize( String region, BigLakeClient client) { this.name = inputName; - this.properties = ImmutableMap.copyOf(properties); - this.projectId = projectId; - this.region = region; + this.bigLakeProperties = ImmutableMap.copyOf(properties); + this.bigLakeProjectId = projectId; + this.bigLakeRegion = region; Preconditions.checkNotNull(client, "BigLake client must not be null"); - this.client = client; + this.bigLakeClient = client; // Users can specify the BigLake catalog ID, otherwise catalog plugin name will be used. // For example, "spark.sql.catalog.=org.apache.iceberg.spark.SparkCatalog" // specifies the plugin name "". - this.catalogId = this.properties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, inputName); + this.catalogId = + this.bigLakeProperties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, inputName); this.catalogName = CatalogName.of(projectId, region, catalogId); String ioImpl = - this.properties.getOrDefault( + this.bigLakeProperties.getOrDefault( CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); - this.io = CatalogUtil.loadFileIO(ioImpl, this.properties, conf); + this.io = CatalogUtil.loadFileIO(ioImpl, this.bigLakeProperties, conf); + + this.closeableGroup = new CloseableGroup(); + closeableGroup.addCloseable(io); + closeableGroup.setSuppressCloseFailure(true); } @Override protected TableOperations newTableOps(TableIdentifier identifier) { return new BigLakeTableOperations( - client, io, name(), tableName(databaseId(identifier.namespace()), identifier.name())); + bigLakeClient, + io, + name(), + tableName(databaseId(identifier.namespace()), identifier.name())); } @Override @@ -156,18 +169,18 @@ public List listTables(Namespace namespace) { ImmutableList dbNames; if (namespace.isEmpty()) { dbNames = - Streams.stream(client.listDatabases(catalogName)) + Streams.stream(bigLakeClient.listDatabases(catalogName)) .map(db -> DatabaseName.parse(db.getName())) .collect(ImmutableList.toImmutableList()); } else { dbNames = ImmutableList.of(databaseName(namespace)); } - ImmutableList.Builder result = new ImmutableList.Builder(); + ImmutableList.Builder result = ImmutableList.builder(); dbNames.stream() .map( dbName -> - Streams.stream(client.listTables(dbName)) + Streams.stream(bigLakeClient.listTables(dbName)) .map(BigLakeCatalog::tableIdentifier) .collect(ImmutableList.toImmutableList())) .forEach(result::addAll); @@ -180,7 +193,7 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. TableMetadata lastMetadata = ops.current(); try { - client.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); + bigLakeClient.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); } catch (NoSuchTableException e) { return false; } @@ -202,14 +215,14 @@ public void renameTable(TableIdentifier from, TableIdentifier to) { "Cannot rename table %s to %s: database must match", from.toString(), to.toString()); - client.renameTable(tableName(fromDbId, from.name()), tableName(toDbId, to.name())); + bigLakeClient.renameTable(tableName(fromDbId, from.name()), tableName(toDbId, to.name())); } @Override public void createNamespace(Namespace namespace, Map metadata) { if (namespace.isEmpty()) { // Used by `CREATE NAMESPACE `. Create a BLMS catalog linked with Iceberg catalog. - client.createCatalog(catalogName, Catalog.getDefaultInstance()); + bigLakeClient.createCatalog(catalogName, Catalog.getDefaultInstance()); LOG.info("Created BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { // Create a database. @@ -220,7 +233,8 @@ public void createNamespace(Namespace namespace, Map metadata) { .putAllParameters(metadata) .setLocationUri(databaseLocation(dbId)); - client.createDatabase(DatabaseName.of(projectId, region, catalogId, dbId), builder.build()); + bigLakeClient.createDatabase( + DatabaseName.of(bigLakeProjectId, bigLakeRegion, catalogId, dbId), builder.build()); } else { throw new IllegalArgumentException( String.format("Invalid namespace (too long): %s", namespace)); @@ -236,7 +250,7 @@ public List listNamespaces(Namespace namespace) { return ImmutableList.of(); } - return Streams.stream(client.listDatabases(catalogName)) + return Streams.stream(bigLakeClient.listDatabases(catalogName)) .map(BigLakeCatalog::namespace) .collect(ImmutableList.toImmutableList()); } @@ -246,14 +260,14 @@ public boolean dropNamespace(Namespace namespace) { try { if (namespace.isEmpty()) { // Used by `DROP NAMESPACE `. Deletes the BLMS catalog linked by Iceberg catalog. - client.deleteCatalog(catalogName); + bigLakeClient.deleteCatalog(catalogName); LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { - client.deleteDatabase(databaseName(namespace)); + bigLakeClient.deleteDatabase(databaseName(namespace)); // Don't delete the data file folder for safety. It aligns with HMS's default behavior. // To support database or catalog level config controlling file deletion in future. } else { - LOG.warn("Invalid namespace (too long): %s", namespace); + LOG.warn("Invalid namespace (too long): {}", namespace); return false; } } catch (NoSuchNamespaceException e) { @@ -268,7 +282,8 @@ public boolean setProperties(Namespace namespace, Map properties HiveDatabaseOptions.Builder optionsBuilder = loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); properties.forEach(optionsBuilder::putParameters); - client.updateDatabaseParameters(databaseName(namespace), optionsBuilder.getParametersMap()); + bigLakeClient.updateDatabaseParameters( + databaseName(namespace), optionsBuilder.getParametersMap()); return true; } @@ -277,7 +292,8 @@ public boolean removeProperties(Namespace namespace, Set properties) { HiveDatabaseOptions.Builder optionsBuilder = loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); properties.forEach(optionsBuilder::removeParameters); - client.updateDatabaseParameters(databaseName(namespace), optionsBuilder.getParametersMap()); + bigLakeClient.updateDatabaseParameters( + databaseName(namespace), optionsBuilder.getParametersMap()); return true; } @@ -285,7 +301,7 @@ public boolean removeProperties(Namespace namespace, Set properties) { public Map loadNamespaceMetadata(Namespace namespace) { if (namespace.isEmpty()) { // Calls getCatalog to check existence. BLMS catalog has no metadata today. - client.getCatalog(catalogName); + bigLakeClient.getCatalog(catalogName); return ImmutableMap.of(); } else if (namespace.levels().length == 1) { return metadata(loadDatabase(namespace)); @@ -301,7 +317,7 @@ public String name() { @Override protected Map properties() { - return properties == null ? ImmutableMap.of() : properties; + return bigLakeProperties == null ? ImmutableMap.of() : bigLakeProperties; } @Override @@ -309,6 +325,11 @@ public void setConf(Object conf) { this.conf = conf; } + @Override + public void close() throws IOException { + closeableGroup.close(); + } + @Override protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { return tableIdentifier.namespace().levels().length == 1; @@ -316,7 +337,8 @@ protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { private String databaseLocation(String dbId) { String warehouseLocation = - LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); + LocationUtil.stripTrailingSlash( + bigLakeProperties.get(CatalogProperties.WAREHOUSE_LOCATION)); Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set"); return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); } @@ -331,7 +353,7 @@ private static Namespace namespace(Database db) { } private TableName tableName(String dbId, String tableId) { - return TableName.of(projectId, region, catalogId, dbId, tableId); + return TableName.of(bigLakeProjectId, bigLakeRegion, catalogId, dbId, tableId); } private String databaseId(Namespace namespace) { @@ -345,11 +367,11 @@ private String databaseId(Namespace namespace) { } private DatabaseName databaseName(Namespace namespace) { - return DatabaseName.of(projectId, region, catalogId, databaseId(namespace)); + return DatabaseName.of(bigLakeProjectId, bigLakeRegion, catalogId, databaseId(namespace)); } private Database loadDatabase(Namespace namespace) { - return client.getDatabase(databaseName(namespace)); + return bigLakeClient.getDatabase(databaseName(namespace)); } private static Map metadata(Database db) { diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index a6df46d83ea4..63c3d473a5c8 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -149,13 +149,13 @@ private void updateTable( tableName()); HiveTableOptions options = table.getHiveOptions(); - String metadataLocationFromMetastore = - options.getParametersOrDefault(METADATA_LOCATION_PROP, ""); - if (metadataLocationFromMetastore.isEmpty()) { + if (!options.containsParameters(METADATA_LOCATION_PROP)) { throw new NoSuchIcebergTableException( "Table %s is not a valid Iceberg table, metadata location is empty", tableName()); } + String metadataLocationFromMetastore = options.getParametersOrThrow(METADATA_LOCATION_PROP); + // If `metadataLocationFromMetastore` is different from metadata location of base, it means // someone has updated metadata location in metastore, which is a conflict update. if (!metadataLocationFromMetastore.equals(oldMetadataLocation)) { @@ -205,7 +205,7 @@ private Table makeNewTable(TableMetadata metadata, String metadataFileLocation) // Follow Iceberg's HiveTableOperations to populate more table parameters for HMS compatibility. private Map buildTableParameters( String metadataFileLocation, TableMetadata metadata) { - ImmutableMap.Builder parameters = new ImmutableMap.Builder(); + ImmutableMap.Builder parameters = ImmutableMap.builder(); parameters.putAll(metadata.properties()); if (metadata.uuid() != null) { parameters.put(TableProperties.UUID, metadata.uuid()); diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index be8a3f03eacd..8e291f669295 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -18,11 +18,8 @@ */ package org.apache.iceberg.gcp.biglake; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.any; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -55,18 +52,14 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; -import org.junit.Rule; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; public class BigLakeCatalogTest extends CatalogTests { - @Rule public final MockitoRule mockito = MockitoJUnit.rule(); - @TempDir public Path temp; + @TempDir private Path temp; private static final String GCP_PROJECT = "my-project"; private static final String GCP_REGION = "us"; @@ -81,7 +74,7 @@ public class BigLakeCatalogTest extends CatalogTests { private BigLakeCatalog bigLakeCatalogUsingMockService; // For tests using a BigLake catalog with a mocked client. - private BigLakeClient mockBigLakeClient; + private BigLakeClient mockBigLakeClient = mock(BigLakeClient.class);; private BigLakeCatalog bigLakeCatalogUsingMockClient; @BeforeEach @@ -118,7 +111,6 @@ public void before() throws Exception { GCP_REGION, new BigLakeClient(settings, GCP_PROJECT, GCP_REGION)); - mockBigLakeClient = mock(BigLakeClient.class); bigLakeCatalogUsingMockClient = new BigLakeCatalog(); bigLakeCatalogUsingMockClient.setConf(new Configuration()); bigLakeCatalogUsingMockClient.initialize( @@ -126,7 +118,9 @@ public void before() throws Exception { } @AfterEach - public void after() { + public void after() throws Exception { + bigLakeCatalogUsingMockService.close(); + bigLakeCatalogUsingMockClient.close(); mockServiceHelper.stop(); } @@ -153,9 +147,10 @@ public void testDefaultWarehouseWithDatabaseLocation_asExpected() { .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) .build()); - assertEquals( - "db_folder/table", - bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + assertThat( + bigLakeCatalogUsingMockClient.defaultWarehouseLocation( + TableIdentifier.of("db", "table"))) + .isEqualTo("db_folder/table"); } @Test @@ -164,21 +159,20 @@ public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { .thenReturn( Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); - assertEquals( - warehouseLocation + "/db.db/table", - bigLakeCatalogUsingMockClient.defaultWarehouseLocation(TableIdentifier.of("db", "table"))); + assertThat( + bigLakeCatalogUsingMockClient.defaultWarehouseLocation( + TableIdentifier.of("db", "table"))) + .isEqualTo(warehouseLocation + "/db.db/table"); } @Test public void testRenameTable_differentDatabase_fail() { - Exception exception = - assertThrows( - IllegalArgumentException.class, + assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.renameTable( - TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))); - assertEquals( - "Cannot rename table db0.t1 to db1.t2: database must match", exception.getMessage()); + TableIdentifier.of("db0", "t1"), TableIdentifier.of("db1", "t2"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot rename table db0.t1 to db1.t2: database must match"); } @Test @@ -190,42 +184,29 @@ public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Excepti } @Test - public void testCreateNamespace_failWhenInvalid() throws Exception { - Exception exception = - assertThrows( - IllegalArgumentException.class, + public void testCreateNamespaceShouldFailWhenInvalid() throws Exception { + assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.createNamespace( - Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())); - assertEquals("Invalid namespace (too long): n0.n1", exception.getMessage()); + Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid namespace (too long): n0.n1"); } @Test - public void testListNamespaces_emptyWhenInvalid() { - assertTrue(bigLakeCatalogUsingMockClient.listNamespaces(Namespace.of("db")).isEmpty()); + public void testListNamespacesShouldReturnEmptyWhenInvalid() { + assertThat(bigLakeCatalogUsingMockClient.listNamespaces(Namespace.of("db"))).isEmpty(); } @Test - public void testDropNamespace_deleteCatalogWhenEmptyNamespace() { + public void testDropNamespaceShouldDeleteCatalogWhenEmptyNamespace() { bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {})); verify(mockBigLakeClient, times(1)) .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } - // BigLake catalog plugin supports dropping a BigLake catalog resource. Spark calls listTables - // with an empty namespace in this case, the purpose is verifying the namespace is empty. We - // check whether there are databases in the BigLake catalog instead. @Test - public void testListTables_emptyNamespace_noDatabase() { - when(mockBigLakeClient.listDatabases(any(CatalogName.class))).thenReturn(ImmutableList.of()); - - assertTrue(bigLakeCatalogUsingMockClient.listTables(Namespace.of()).isEmpty()); - verify(mockBigLakeClient, times(1)) - .listDatabases(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); - } - - @Test - public void testListTables_emptyNamespace_listTablesInAllDbs() { + public void testListTablesShouldListTablesInAllDbsWhenNamespaceIsEmpty() { DatabaseName db1Name = DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db1"); DatabaseName db2Name = DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db2"); @@ -248,35 +229,36 @@ public void testListTables_emptyNamespace_listTablesInAllDbs() { .thenReturn(ImmutableList.of(Table.newBuilder().setName(table3Name.toString()).build())); List result = bigLakeCatalogUsingMockClient.listTables(Namespace.of()); - assertEquals(3, result.size()); - assertEquals(TableIdentifier.of("db1", "tbl1"), result.get(0)); - assertEquals(TableIdentifier.of("db1", "tbl2"), result.get(1)); - assertEquals(TableIdentifier.of("db2", "tbl3"), result.get(2)); + assertThat(result) + .containsExactlyInAnyOrder( + TableIdentifier.of("db1", "tbl1"), + TableIdentifier.of("db1", "tbl2"), + TableIdentifier.of("db2", "tbl3")); } @Test - public void testDropNamespace_failWhenInvalid() throws Exception { - assertFalse( - bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))); + public void testDropNamespaceShouldFailWhenInvalid() throws Exception { + assertThat(bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))) + .isFalse(); } @Test - public void testSetProperties_failWhenNamespacesAreInvalid() throws Exception { - assertThrows( - NoSuchNamespaceException.class, - () -> - bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {}), ImmutableMap.of())); + public void testSetPropertiesShouldFailWhenNamespacesAreInvalid() throws Exception { + assertThatThrownBy( + () -> + bigLakeCatalogUsingMockClient.setProperties( + Namespace.of(new String[] {}), ImmutableMap.of())) + .isInstanceOf(NoSuchNamespaceException.class); - assertThrows( - NoSuchNamespaceException.class, - () -> - bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())); + assertThatThrownBy( + () -> + bigLakeCatalogUsingMockClient.setProperties( + Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())) + .isInstanceOf(NoSuchNamespaceException.class); } @Test - public void testSetProperties_succeedForDatabase() throws Exception { + public void testSetPropertiesShouldSucceedForDatabase() throws Exception { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -286,10 +268,11 @@ public void testSetProperties_succeedForDatabase() throws Exception { .putParameters("key2", "value2")) .build()); - assertTrue( - bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {"db"}), - ImmutableMap.of("key2", "value222", "key3", "value3"))); + assertThat( + bigLakeCatalogUsingMockClient.setProperties( + Namespace.of(new String[] {"db"}), + ImmutableMap.of("key2", "value222", "key3", "value3"))) + .isTrue(); verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), @@ -297,22 +280,22 @@ public void testSetProperties_succeedForDatabase() throws Exception { } @Test - public void testRemoveProperties_failWhenNamespacesAreInvalid() throws Exception { - assertThrows( - NoSuchNamespaceException.class, - () -> - bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {}), ImmutableSet.of())); + public void testRemovePropertiesShouldFailWhenNamespacesAreInvalid() throws Exception { + assertThatThrownBy( + () -> + bigLakeCatalogUsingMockClient.removeProperties( + Namespace.of(new String[] {}), ImmutableSet.of())) + .isInstanceOf(NoSuchNamespaceException.class); - assertThrows( - NoSuchNamespaceException.class, - () -> - bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())); + assertThatThrownBy( + () -> + bigLakeCatalogUsingMockClient.removeProperties( + Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())) + .isInstanceOf(NoSuchNamespaceException.class); } @Test - public void testRemoveProperties_succeedForDatabase() throws Exception { + public void testRemovePropertiesShouldSucceedForDatabase() throws Exception { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -322,9 +305,10 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { .putParameters("key2", "value2")) .build()); - assertTrue( - bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))); + assertThat( + bigLakeCatalogUsingMockClient.removeProperties( + Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))) + .isTrue(); verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"), @@ -332,17 +316,15 @@ public void testRemoveProperties_succeedForDatabase() throws Exception { } @Test - public void testLoadNamespaceMetadata_catalogAsExpected() throws Exception { - assertTrue( - bigLakeCatalogUsingMockClient - .loadNamespaceMetadata(Namespace.of(new String[] {})) - .isEmpty()); + public void testLoadNamespaceMetadataAsExpectedForCatalogs() throws Exception { + assertThat(bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of(new String[] {}))) + .isEmpty(); verify(mockBigLakeClient, times(1)) .getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @Test - public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { + public void testLoadNamespaceMetadataAsExpectedForDatabases() throws Exception { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -353,55 +335,30 @@ public void testLoadNamespaceMetadata_databaseAsExpected() throws Exception { .putParameters("key2", "value2")) .build()); - assertEquals( - ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2"), - bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))); + assertThat( + bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))) + .containsAllEntriesOf( + ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2")); } @Test - public void testLoadNamespaceMetadata_failWhenInvalid() throws Exception { - Exception exception = - assertThrows( - NoSuchNamespaceException.class, + public void testLoadNamespaceMetadataShouldFailWhenInvalid() throws Exception { + assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.loadNamespaceMetadata( - Namespace.of(new String[] {"n0", "n1"}))); - assertEquals("Namespace does not exist: n0.n1", exception.getMessage()); - } - - @Test - public void testSetBigLakeCatalogInProperties_asExpected() throws Exception { - BigLakeCatalog catalog = new BigLakeCatalog(); - catalog.initialize( - CATALOG_ID, - /* properties= */ ImmutableMap.of( - GCPProperties.BIGLAKE_PROJECT_ID, - GCP_PROJECT, - CatalogProperties.WAREHOUSE_LOCATION, - warehouseLocation, - GCPProperties.BIGLAKE_CATALOG_ID, - "customized_catalog"), - GCP_PROJECT, - GCP_REGION, - mockBigLakeClient); - - catalog.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); - verify(mockBigLakeClient, times(1)) - .createCatalog( - CatalogName.of(GCP_PROJECT, GCP_REGION, "customized_catalog"), - Catalog.getDefaultInstance()); + Namespace.of(new String[] {"n0", "n1"}))) + .isInstanceOf(NoSuchNamespaceException.class) + .hasMessage("Namespace does not exist: n0.n1"); } @Test - public void testNewTableOps_failedForInvalidNamespace() throws Exception { - Exception exception = - assertThrows( - NoSuchNamespaceException.class, + public void testNewTableOpsShouldfailedForInvalidNamespace() throws Exception { + assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.newTableOps( - TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))); - assertEquals( - "BigLake database namespace must use format ., invalid namespace: n0.n1", - exception.getMessage()); + TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))) + .isInstanceOf(NoSuchNamespaceException.class) + .hasMessage( + "BigLake database namespace must use format ., invalid namespace: n0.n1"); } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 9a293dff4727..6de11003eb1f 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -19,10 +19,10 @@ package org.apache.iceberg.gcp.biglake; import static org.apache.iceberg.types.Types.NestedField.required; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -37,6 +37,7 @@ import io.grpc.Status.Code; import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.Optional; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.TrueFileFilter; @@ -49,19 +50,15 @@ import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; public class BigLakeTableOperationsTest { - @Rule public final MockitoRule mockito = MockitoJUnit.rule(); - @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); + @TempDir private Path temp; private static final String CATALOG_NAME = "iceberg"; @@ -79,15 +76,14 @@ public class BigLakeTableOperationsTest { required(1, "id", Types.IntegerType.get(), "unique ID"), required(2, "data", Types.StringType.get())); - @Mock private BigLakeClient bigLakeClient; - + private BigLakeClient bigLakeClient = mock(BigLakeClient.class); private BigLakeCatalog bigLakeCatalog; private String warehouseLocation; private BigLakeTableOperations tableOps; - @Before + @BeforeEach public void before() throws Exception { - warehouseLocation = tempFolder.newFolder("hive-warehouse").toString(); + warehouseLocation = temp.toFile().getAbsolutePath(); ImmutableMap properties = ImmutableMap.of( GCPProperties.BIGLAKE_PROJECT_ID, @@ -103,8 +99,13 @@ public void before() throws Exception { this.tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(SPARK_TABLE_ID); } + @AfterEach + public void after() throws Exception { + bigLakeCatalog.close(); + } + @Test - public void testDoCommit_useEtagForUpdateTable() throws Exception { + public void testDoCommitShouldUseEtagForUpdateTable() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) .thenThrow(new NoSuchTableException("error message getTable")); Table createdTable = createTestTable(); @@ -122,12 +123,12 @@ public void testDoCommit_useEtagForUpdateTable() throws Exception { ArgumentCaptor etagCaptor = ArgumentCaptor.forClass(String.class); verify(bigLakeClient, times(1)) .updateTableParameters(nameCaptor.capture(), any(), etagCaptor.capture()); - assertEquals(TABLE_NAME, nameCaptor.getValue()); - assertEquals("etag", etagCaptor.getValue()); + assertThat(nameCaptor.getValue()).isEqualTo(TABLE_NAME); + assertThat(etagCaptor.getValue()).isEqualTo("etag"); } @Test - public void testDoCommit_failWhenEtagMismatch() throws Exception { + public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) .thenThrow(new NoSuchTableException("error message getTable")); Table createdTable = createTestTable(); @@ -144,28 +145,29 @@ public void testDoCommit_failWhenEtagMismatch() throws Exception { new RuntimeException("error message etag mismatch"), GrpcStatusCode.of(Code.ABORTED), false)); - assertThrows( - CommitFailedException.class, - () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()); + + assertThatThrownBy( + () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()) + .isInstanceOf(CommitFailedException.class); } @Test - public void testDoFresh_refreshReturnNullForNonIcebergTable() throws Exception { + public void testDoFreshRefreshShouldReturnNullForNonIcebergTable() throws Exception { when(bigLakeClient.getTable(TABLE_NAME)) .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); - assertEquals(null, tableOps.refresh()); + assertThat(tableOps.refresh()).isNull(); } @Test public void testTableName() throws Exception { - assertEquals(tableOps.tableName(), "iceberg.db.tbl"); + assertThat(tableOps.tableName()).isEqualTo("iceberg.db.tbl"); } private Table createTestTable() throws IOException { TableIdentifier tableIdent = TableIdentifier.of(TABLE_NAME.getDatabase(), TABLE_NAME.getTable()); - String tableDir = tempFolder.newFolder(TABLE_NAME.getTable()).toString(); + String tableDir = new File(warehouseLocation, TABLE_NAME.getTable()).getAbsolutePath(); bigLakeCatalog .buildTable(tableIdent, SCHEMA) @@ -174,7 +176,7 @@ private Table createTestTable() throws IOException { .commitTransaction(); Optional metadataLocation = getAnyIcebergMetadataFilePath(tableDir); - assertTrue(metadataLocation.isPresent()); + assertThat(metadataLocation).isPresent(); return Table.newBuilder() .setName(TABLE_NAME.toString()) .setHiveOptions( From 79ddc3dd067a28161e00f2d78fc257ed3916a228 Mon Sep 17 00:00:00 2001 From: coufon Date: Sun, 16 Jul 2023 04:59:04 +0000 Subject: [PATCH 15/22] fix more review comments --- build.gradle | 15 --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 35 +++-- .../iceberg/gcp/biglake/BigLakeClient.java | 18 +-- .../gcp/biglake/BigLakeTableOperations.java | 7 +- .../gcp/biglake/BigLakeCatalogTest.java | 120 ++++++++---------- .../biglake/BigLakeTableOperationsTest.java | 19 ++- 6 files changed, 95 insertions(+), 119 deletions(-) diff --git a/build.gradle b/build.gradle index 768926829366..274893704c8f 100644 --- a/build.gradle +++ b/build.gradle @@ -615,21 +615,6 @@ project(':iceberg-gcp') { exclude group: 'javax.servlet', module: 'servlet-api' exclude group: 'com.google.code.gson', module: 'gson' } - compileOnly("org.apache.hive:hive-metastore") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.pentaho' // missing dependency - exclude group: 'org.apache.hbase' - exclude group: 'org.apache.logging.log4j' - exclude group: 'co.cask.tephra' - exclude group: 'com.google.code.findbugs', module: 'jsr305' - exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all' - exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet' - exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle' - exclude group: 'com.tdunning', module: 'json' - exclude group: 'javax.transaction', module: 'transaction-api' - exclude group: 'com.zaxxer', module: 'HikariCP' - } testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 1abcbe8342fe..259e4c6c2002 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -103,9 +103,7 @@ public void initialize(String inputName, Map properties) { client = new BigLakeClient( properties.getOrDefault( - GCPProperties.BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT), - projectId, - region); + GCPProperties.BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT)); } catch (IOException e) { throw new ServiceFailureException(e, "Creating BigLake client failed"); } @@ -189,16 +187,21 @@ public List listTables(Namespace namespace) { @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { - TableOperations ops = newTableOps(identifier); - // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. - TableMetadata lastMetadata = ops.current(); + TableOperations ops = null; + TableMetadata lastMetadata = null; + if (purge) { + ops = newTableOps(identifier); + // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. + lastMetadata = ops.current(); + } + try { bigLakeClient.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); } catch (NoSuchTableException e) { return false; } - if (purge && lastMetadata != null) { + if (ops != null && lastMetadata != null) { CatalogUtil.dropTableData(ops.io(), lastMetadata); } @@ -245,8 +248,8 @@ public void createNamespace(Namespace namespace, Map metadata) { public List listNamespaces(Namespace namespace) { if (!namespace.isEmpty()) { // BLMS does not support namespaces under database or tables, returns empty. - // It is called when dropping a namespace to make sure it's empty (listTables is called as - // well), returns empty to unblock deletion. + // It is called when dropping a namespace to make sure it's empty, returns empty to unblock + // deletion. return ImmutableList.of(); } @@ -327,7 +330,9 @@ public void setConf(Object conf) { @Override public void close() throws IOException { - closeableGroup.close(); + if (closeableGroup != null) { + closeableGroup.close(); + } } @Override @@ -359,8 +364,11 @@ private TableName tableName(String dbId, String tableId) { private String databaseId(Namespace namespace) { if (namespace.levels().length != 1) { throw new NoSuchNamespaceException( - "BigLake database namespace must use format ., invalid namespace: %s", - namespace); + namespace.isEmpty() + ? "Invalid BigLake database namespace: empty" + : String.format( + "BigLake database namespace must use format ., invalid namespace: %s", + namespace)); } return namespace.level(0); @@ -377,7 +385,8 @@ private Database loadDatabase(Namespace namespace) { private static Map metadata(Database db) { HiveDatabaseOptions options = db.getHiveOptions(); return new ImmutableMap.Builder() - .putAll(options.getParameters()) + .putAll(options.getParametersMap()) + // Add the storage location of the database to metadata. .put("location", options.getLocationUri()) .build(); } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index de20f9f77d55..b088c14bd134 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -55,21 +55,14 @@ /** A client of Google BigLake service. */ final class BigLakeClient { - private final String projectId; - private final String region; private final MetastoreServiceClient stub; /** * Constructs a client of Google BigLake Service. * * @param settings BigLake service settings - * @param projectId GCP project ID - * @param region GCP region supported by BigLake, e.g., "us" */ - BigLakeClient(MetastoreServiceSettings settings, String projectId, String region) - throws IOException { - this.projectId = projectId; - this.region = region; + BigLakeClient(MetastoreServiceSettings settings) throws IOException { this.stub = MetastoreServiceClient.create(settings); } @@ -77,14 +70,9 @@ final class BigLakeClient { * Constructs a client of Google BigLake Service. * * @param biglakeEndpoint BigLake service gRPC endpoint, e.g., "biglake.googleapis.com:443" - * @param projectId GCP project ID - * @param region GCP region supported by BigLake, e.g., "us" */ - BigLakeClient(String biglakeEndpoint, String projectId, String region) throws IOException { - this( - MetastoreServiceSettings.newBuilder().setEndpoint(biglakeEndpoint).build(), - projectId, - region); + BigLakeClient(String biglakeEndpoint) throws IOException { + this(MetastoreServiceSettings.newBuilder().setEndpoint(biglakeEndpoint).build()); } public Catalog createCatalog(CatalogName name, Catalog catalog) { diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 63c3d473a5c8..52f9763d1caf 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -46,7 +46,7 @@ public final class BigLakeTableOperations extends BaseMetastoreTableOperations { private final BigLakeClient client; private final FileIO io; - // The catalog name. + // The name of this Iceberg catalog plugin: spark.sql.catalog.. private final String name; private final TableName tableName; @@ -96,7 +96,10 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } commitStatus = CommitStatus.SUCCESS; - } catch (AlreadyExistsException | CommitFailedException | CommitStateUnknownException e) { + } catch (AlreadyExistsException | CommitFailedException e) { + throw e; + } catch (CommitStateUnknownException e) { + commitStatus = CommitStatus.UNKNOWN; throw e; } catch (Throwable e) { commitStatus = checkCommitStatus(newMetadataLocation, metadata); diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 8e291f669295..6ed504857906 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -26,7 +26,6 @@ import static org.mockito.Mockito.when; import com.google.api.gax.core.NoCredentialsProvider; -import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; import com.google.cloud.bigquery.biglake.v1.Catalog; @@ -37,7 +36,6 @@ import com.google.cloud.bigquery.biglake.v1.MetastoreServiceSettings; import com.google.cloud.bigquery.biglake.v1.Table; import com.google.cloud.bigquery.biglake.v1.TableName; -import java.io.File; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -65,12 +63,8 @@ public class BigLakeCatalogTest extends CatalogTests { private static final String GCP_REGION = "us"; private static final String CATALOG_ID = "biglake"; - private String warehouseLocation; - // For tests using a BigLake catalog connecting to a mocked service. - private MockMetastoreService mockMetastoreService; private MockServiceHelper mockServiceHelper; - private LocalChannelProvider channelProvider; private BigLakeCatalog bigLakeCatalogUsingMockService; // For tests using a BigLake catalog with a mocked client. @@ -79,37 +73,29 @@ public class BigLakeCatalogTest extends CatalogTests { @BeforeEach public void before() throws Exception { - mockMetastoreService = new MockMetastoreService(); mockServiceHelper = new MockServiceHelper( - UUID.randomUUID().toString(), Arrays.asList(mockMetastoreService)); + UUID.randomUUID().toString(), + Arrays.asList(new MockMetastoreService())); mockServiceHelper.start(); - File warehouse = temp.toFile(); - warehouseLocation = warehouse.getAbsolutePath(); - ImmutableMap properties = ImmutableMap.of( GCPProperties.BIGLAKE_PROJECT_ID, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, - warehouseLocation); + temp.toAbsolutePath().toString()); - channelProvider = mockServiceHelper.createChannelProvider(); MetastoreServiceSettings settings = MetastoreServiceSettings.newBuilder() - .setTransportChannelProvider(channelProvider) + .setTransportChannelProvider(mockServiceHelper.createChannelProvider()) .setCredentialsProvider(NoCredentialsProvider.create()) .build(); bigLakeCatalogUsingMockService = new BigLakeCatalog(); bigLakeCatalogUsingMockService.setConf(new Configuration()); bigLakeCatalogUsingMockService.initialize( - CATALOG_ID, - properties, - GCP_PROJECT, - GCP_REGION, - new BigLakeClient(settings, GCP_PROJECT, GCP_REGION)); + CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, new BigLakeClient(settings)); bigLakeCatalogUsingMockClient = new BigLakeCatalog(); bigLakeCatalogUsingMockClient.setConf(new Configuration()); @@ -119,9 +105,17 @@ public void before() throws Exception { @AfterEach public void after() throws Exception { - bigLakeCatalogUsingMockService.close(); - bigLakeCatalogUsingMockClient.close(); - mockServiceHelper.stop(); + if (bigLakeCatalogUsingMockService != null) { + bigLakeCatalogUsingMockService.close(); + } + + if (bigLakeCatalogUsingMockClient != null) { + bigLakeCatalogUsingMockClient.close(); + } + + if (mockServiceHelper != null) { + mockServiceHelper.stop(); + } } @Override @@ -140,7 +134,7 @@ protected boolean supportsNamesWithSlashes() { } @Test - public void testDefaultWarehouseWithDatabaseLocation_asExpected() { + public void testDefaultWarehouseWithDatabaseLocation() { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -154,7 +148,7 @@ public void testDefaultWarehouseWithDatabaseLocation_asExpected() { } @Test - public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { + public void testDefaultWarehouseWithoutDatabaseLocation() { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) .thenReturn( Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); @@ -162,11 +156,11 @@ public void testDefaultWarehouseWithoutDatabaseLocation_asExpected() { assertThat( bigLakeCatalogUsingMockClient.defaultWarehouseLocation( TableIdentifier.of("db", "table"))) - .isEqualTo(warehouseLocation + "/db.db/table"); + .isEqualTo(temp.toAbsolutePath().toString() + "/db.db/table"); } @Test - public void testRenameTable_differentDatabase_fail() { + public void testRenameTableToDifferentDatabaseShouldFail() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.renameTable( @@ -176,31 +170,31 @@ public void testRenameTable_differentDatabase_fail() { } @Test - public void testCreateNamespace_createCatalogWhenEmptyNamespace() throws Exception { - bigLakeCatalogUsingMockClient.createNamespace(Namespace.of(new String[] {}), ImmutableMap.of()); + public void testCreateNamespaceShouldCreateCatalogWhenNamespaceIsEmpty() { + bigLakeCatalogUsingMockClient.createNamespace(Namespace.empty(), ImmutableMap.of()); verify(mockBigLakeClient, times(1)) .createCatalog( CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID), Catalog.getDefaultInstance()); } @Test - public void testCreateNamespaceShouldFailWhenInvalid() throws Exception { + public void testCreateNamespaceShouldFailWhenInvalid() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.createNamespace( - Namespace.of(new String[] {"n0", "n1"}), ImmutableMap.of())) + Namespace.of("n0", "n1"), ImmutableMap.of())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid namespace (too long): n0.n1"); } @Test - public void testListNamespacesShouldReturnEmptyWhenInvalid() { + public void testListNamespacesShouldReturnEmptyWhenNamespaceIsNotEmpty() { assertThat(bigLakeCatalogUsingMockClient.listNamespaces(Namespace.of("db"))).isEmpty(); } @Test public void testDropNamespaceShouldDeleteCatalogWhenEmptyNamespace() { - bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {})); + bigLakeCatalogUsingMockClient.dropNamespace(Namespace.empty()); verify(mockBigLakeClient, times(1)) .deleteCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @@ -228,7 +222,7 @@ public void testListTablesShouldListTablesInAllDbsWhenNamespaceIsEmpty() { when(mockBigLakeClient.listTables(db2Name)) .thenReturn(ImmutableList.of(Table.newBuilder().setName(table3Name.toString()).build())); - List result = bigLakeCatalogUsingMockClient.listTables(Namespace.of()); + List result = bigLakeCatalogUsingMockClient.listTables(Namespace.empty()); assertThat(result) .containsExactlyInAnyOrder( TableIdentifier.of("db1", "tbl1"), @@ -237,28 +231,28 @@ public void testListTablesShouldListTablesInAllDbsWhenNamespaceIsEmpty() { } @Test - public void testDropNamespaceShouldFailWhenInvalid() throws Exception { - assertThat(bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of(new String[] {"n0", "n1"}))) - .isFalse(); + public void testDropNamespaceShouldFailWhenNamespaceIsTooLong() { + assertThat(bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of("n0", "n1"))).isFalse(); } @Test - public void testSetPropertiesShouldFailWhenNamespacesAreInvalid() throws Exception { + public void testSetPropertiesShouldFailWhenNamespacesAreInvalid() { assertThatThrownBy( - () -> - bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {}), ImmutableMap.of())) - .isInstanceOf(NoSuchNamespaceException.class); + () -> bigLakeCatalogUsingMockClient.setProperties(Namespace.empty(), ImmutableMap.of())) + .isInstanceOf(NoSuchNamespaceException.class) + .hasMessage("Invalid BigLake database namespace: empty"); assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {"db", "tbl"}), ImmutableMap.of())) - .isInstanceOf(NoSuchNamespaceException.class); + Namespace.of("db", "tbl"), ImmutableMap.of())) + .isInstanceOf(NoSuchNamespaceException.class) + .hasMessage( + "BigLake database namespace must use format ., invalid namespace: db.tbl"); } @Test - public void testSetPropertiesShouldSucceedForDatabase() throws Exception { + public void testSetPropertiesShouldSucceedForDatabase() { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -270,8 +264,7 @@ public void testSetPropertiesShouldSucceedForDatabase() throws Exception { assertThat( bigLakeCatalogUsingMockClient.setProperties( - Namespace.of(new String[] {"db"}), - ImmutableMap.of("key2", "value222", "key3", "value3"))) + Namespace.of("db"), ImmutableMap.of("key2", "value222", "key3", "value3"))) .isTrue(); verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( @@ -280,22 +273,25 @@ public void testSetPropertiesShouldSucceedForDatabase() throws Exception { } @Test - public void testRemovePropertiesShouldFailWhenNamespacesAreInvalid() throws Exception { + public void testRemovePropertiesShouldFailWhenNamespacesAreInvalid() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {}), ImmutableSet.of())) - .isInstanceOf(NoSuchNamespaceException.class); + Namespace.empty(), ImmutableSet.of())) + .isInstanceOf(NoSuchNamespaceException.class) + .hasMessage("Invalid BigLake database namespace: empty"); assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {"db", "tbl"}), ImmutableSet.of())) - .isInstanceOf(NoSuchNamespaceException.class); + Namespace.of("db", "tbl"), ImmutableSet.of())) + .isInstanceOf(NoSuchNamespaceException.class) + .hasMessage( + "BigLake database namespace must use format ., invalid namespace: db.tbl"); } @Test - public void testRemovePropertiesShouldSucceedForDatabase() throws Exception { + public void testRemovePropertiesShouldSucceedForDatabase() { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -307,7 +303,7 @@ public void testRemovePropertiesShouldSucceedForDatabase() throws Exception { assertThat( bigLakeCatalogUsingMockClient.removeProperties( - Namespace.of(new String[] {"db"}), ImmutableSet.of("key1", "key3"))) + Namespace.of("db"), ImmutableSet.of("key1", "key3"))) .isTrue(); verify(mockBigLakeClient, times(1)) .updateDatabaseParameters( @@ -316,15 +312,14 @@ public void testRemovePropertiesShouldSucceedForDatabase() throws Exception { } @Test - public void testLoadNamespaceMetadataAsExpectedForCatalogs() throws Exception { - assertThat(bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of(new String[] {}))) - .isEmpty(); + public void testEmptyNamespaceLoadsCatalogMetadata() { + assertThat(bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.empty())).isEmpty(); verify(mockBigLakeClient, times(1)) .getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @Test - public void testLoadNamespaceMetadataAsExpectedForDatabases() throws Exception { + public void testLoadNamespaceMetadataForDatabases() { when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() @@ -335,24 +330,21 @@ public void testLoadNamespaceMetadataAsExpectedForDatabases() throws Exception { .putParameters("key2", "value2")) .build()); - assertThat( - bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of(new String[] {"db"}))) + assertThat(bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of("db"))) .containsAllEntriesOf( ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2")); } @Test - public void testLoadNamespaceMetadataShouldFailWhenInvalid() throws Exception { + public void testLoadNamespaceMetadataShouldFailWhenInvalid() { assertThatThrownBy( - () -> - bigLakeCatalogUsingMockClient.loadNamespaceMetadata( - Namespace.of(new String[] {"n0", "n1"}))) + () -> bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of("n0", "n1"))) .isInstanceOf(NoSuchNamespaceException.class) .hasMessage("Namespace does not exist: n0.n1"); } @Test - public void testNewTableOpsShouldfailedForInvalidNamespace() throws Exception { + public void testNewTableOpsShouldfailedForInvalidNamespace() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.newTableOps( diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 6de11003eb1f..bb05699836d4 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -69,7 +69,7 @@ public class BigLakeTableOperationsTest { private static final String TABLE_ID = "tbl"; private static final TableName TABLE_NAME = TableName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, DB_ID, TABLE_ID); - private static final TableIdentifier SPARK_TABLE_ID = TableIdentifier.of(DB_ID, TABLE_ID); + private static final TableIdentifier TABLE_IDENTIFIER = TableIdentifier.of(DB_ID, TABLE_ID); private static final Schema SCHEMA = new Schema( @@ -78,25 +78,23 @@ public class BigLakeTableOperationsTest { private BigLakeClient bigLakeClient = mock(BigLakeClient.class); private BigLakeCatalog bigLakeCatalog; - private String warehouseLocation; private BigLakeTableOperations tableOps; @BeforeEach public void before() throws Exception { - warehouseLocation = temp.toFile().getAbsolutePath(); ImmutableMap properties = ImmutableMap.of( GCPProperties.BIGLAKE_PROJECT_ID, GCP_PROJECT, CatalogProperties.WAREHOUSE_LOCATION, - warehouseLocation, + temp.toFile().getAbsolutePath(), GCPProperties.BIGLAKE_CATALOG_ID, CATALOG_ID); bigLakeCatalog = new BigLakeCatalog(); bigLakeCatalog.setConf(new Configuration()); bigLakeCatalog.initialize(CATALOG_NAME, properties, GCP_PROJECT, GCP_REGION, bigLakeClient); - this.tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(SPARK_TABLE_ID); + tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(TABLE_IDENTIFIER); } @AfterEach @@ -114,7 +112,7 @@ public void testDoCommitShouldUseEtagForUpdateTable() throws Exception { reset(bigLakeClient); when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); - org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(SPARK_TABLE_ID); + org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(TABLE_IDENTIFIER); when(bigLakeClient.updateTableParameters(any(), any(), any())).thenReturn(tableWithEtag); loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit(); @@ -137,7 +135,7 @@ public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { reset(bigLakeClient); when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); - org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(SPARK_TABLE_ID); + org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(TABLE_IDENTIFIER); when(bigLakeClient.updateTableParameters(any(), any(), any())) .thenThrow( @@ -152,7 +150,7 @@ public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { } @Test - public void testDoFreshRefreshShouldReturnNullForNonIcebergTable() throws Exception { + public void testDoFreshRefreshShouldReturnNullForNonIcebergTable() { when(bigLakeClient.getTable(TABLE_NAME)) .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); @@ -160,14 +158,15 @@ public void testDoFreshRefreshShouldReturnNullForNonIcebergTable() throws Except } @Test - public void testTableName() throws Exception { + public void testTableName() { assertThat(tableOps.tableName()).isEqualTo("iceberg.db.tbl"); } private Table createTestTable() throws IOException { TableIdentifier tableIdent = TableIdentifier.of(TABLE_NAME.getDatabase(), TABLE_NAME.getTable()); - String tableDir = new File(warehouseLocation, TABLE_NAME.getTable()).getAbsolutePath(); + String tableDir = + new File(temp.toFile().getAbsolutePath(), TABLE_NAME.getTable()).getAbsolutePath(); bigLakeCatalog .buildTable(tableIdent, SCHEMA) From 467b3bb3d15af380b85b9d8b3d41ea892968e73d Mon Sep 17 00:00:00 2001 From: coufon Date: Sun, 16 Jul 2023 23:15:20 +0000 Subject: [PATCH 16/22] fix more review comments. --- .../org/apache/iceberg/gcp/GCPProperties.java | 12 +- .../iceberg/gcp/biglake/BigLakeCatalog.java | 205 +++++++++--------- .../gcp/biglake/BigLakeTableOperations.java | 2 +- .../gcp/biglake/BigLakeCatalogTest.java | 24 +- .../biglake/BigLakeTableOperationsTest.java | 6 +- 5 files changed, 125 insertions(+), 124 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java b/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java index 700571f35c2a..af04032c5261 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java @@ -37,18 +37,16 @@ public class GCPProperties implements Serializable { public static final String GCS_CHANNEL_READ_CHUNK_SIZE = "gcs.channel.read.chunk-size-bytes"; public static final String GCS_CHANNEL_WRITE_CHUNK_SIZE = "gcs.channel.write.chunk-size-bytes"; - // For BigQuery BigLake Metastore. - // The endpoint of BigLake API. - // Optional, default to BigLakeCatalog.DEFAULT_BIGLAKE_SERVICE_ENDPOINT. - public static final String BIGLAKE_ENDPOINT = "biglake.endpoint"; // The GCP project ID. Required. - public static final String BIGLAKE_PROJECT_ID = "biglake.project-id"; + public static final String PROJECT_ID = "project-id"; + // The GCP region (https://cloud.google.com/bigquery/docs/locations). Required. - public static final String BIGLAKE_GCP_REGION = "biglake.region"; + public static final String REGION = "region"; + // The BigLake Metastore catalog ID. It is the container resource of databases and tables. // It links a BLMS catalog with this Iceberg catalog. // Optional, default to the Spark catalog plugin name. - public static final String BIGLAKE_CATALOG_ID = "biglake.catalog-id"; + public static final String BIGLAKE_CATALOG_ID = "catalog-id"; private String projectId; private String clientLibToken; diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 259e4c6c2002..94a2017dbc26 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -64,18 +64,18 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog private static final Logger LOG = LoggerFactory.getLogger(BigLakeCatalog.class); - // The name of this Iceberg catalog plugin: spark.sql.catalog.. + // The name of this Iceberg catalog plugin. private String name; - private Map bigLakeProperties; + private Map properties; private FileIO io; private Object conf; - private String bigLakeProjectId; - private String bigLakeRegion; + private String projectId; + private String region; // BLMS catalog ID and fully qualified name. private String catalogId; private CatalogName catalogName; - private BigLakeClient bigLakeClient; + private BigLakeClient client; private CloseableGroup closeableGroup; @@ -85,57 +85,48 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog public BigLakeCatalog() {} @Override - public void initialize(String inputName, Map properties) { - Preconditions.checkArgument( - properties.containsKey(GCPProperties.BIGLAKE_PROJECT_ID), - "GCP project ID must be specified"); - String projectId = properties.get(GCPProperties.BIGLAKE_PROJECT_ID); - - Preconditions.checkArgument( - properties.containsKey(GCPProperties.BIGLAKE_GCP_REGION), "GCP region must be specified"); - String region = properties.get(GCPProperties.BIGLAKE_GCP_REGION); - - BigLakeClient client; + public void initialize(String initName, Map initProperties) { + BigLakeClient initClient; try { + // CatalogProperties.URI specifies the endpoint of BigLake API. // TODO: to add more auth options of the client. Currently it uses default auth // (https://github.com/googleapis/google-cloud-java#application-default-credentials) // that works on GCP services (e.g., GCE, GKE, Dataproc). - client = + initClient = new BigLakeClient( - properties.getOrDefault( - GCPProperties.BIGLAKE_ENDPOINT, DEFAULT_BIGLAKE_SERVICE_ENDPOINT)); + initProperties.getOrDefault(CatalogProperties.URI, DEFAULT_BIGLAKE_SERVICE_ENDPOINT)); } catch (IOException e) { throw new ServiceFailureException(e, "Creating BigLake client failed"); } - initialize(inputName, properties, projectId, region, client); + initialize(initName, initProperties, initClient); } @VisibleForTesting - void initialize( - String inputName, - Map properties, - String projectId, - String region, - BigLakeClient client) { - this.name = inputName; - this.bigLakeProperties = ImmutableMap.copyOf(properties); - this.bigLakeProjectId = projectId; - this.bigLakeRegion = region; - Preconditions.checkNotNull(client, "BigLake client must not be null"); - this.bigLakeClient = client; + void initialize(String initName, Map initProperties, BigLakeClient initClient) { + this.name = initName; + this.properties = ImmutableMap.copyOf(initProperties); + + Preconditions.checkNotNull(initClient, "BigLake client must not be null"); + this.client = initClient; + + Preconditions.checkArgument( + properties.containsKey(GCPProperties.PROJECT_ID), "GCP project ID must be specified"); + this.projectId = properties.get(GCPProperties.PROJECT_ID); + + Preconditions.checkArgument( + properties.containsKey(GCPProperties.REGION), "GCP region must be specified"); + this.region = properties.get(GCPProperties.REGION); // Users can specify the BigLake catalog ID, otherwise catalog plugin name will be used. // For example, "spark.sql.catalog.=org.apache.iceberg.spark.SparkCatalog" // specifies the plugin name "". - this.catalogId = - this.bigLakeProperties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, inputName); + this.catalogId = properties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, initName); this.catalogName = CatalogName.of(projectId, region, catalogId); String ioImpl = - this.bigLakeProperties.getOrDefault( - CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); - this.io = CatalogUtil.loadFileIO(ioImpl, this.bigLakeProperties, conf); + properties.getOrDefault(CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); + this.io = CatalogUtil.loadFileIO(ioImpl, properties, conf); this.closeableGroup = new CloseableGroup(); closeableGroup.addCloseable(io); @@ -144,41 +135,46 @@ void initialize( @Override protected TableOperations newTableOps(TableIdentifier identifier) { - return new BigLakeTableOperations( - bigLakeClient, - io, - name(), - tableName(databaseId(identifier.namespace()), identifier.name())); + String db = databaseId(identifier.namespace()); + if (db == null) { + throwInvalidDbNamespaceError(identifier.namespace()); + } + + return new BigLakeTableOperations(client, io, name(), tableName(db, identifier.name())); } @Override protected String defaultWarehouseLocation(TableIdentifier identifier) { + String db = databaseId(identifier.namespace()); + if (db == null) { + throwInvalidDbNamespaceError(identifier.namespace()); + } + String locationUri = loadDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); return String.format( "%s/%s", - Strings.isNullOrEmpty(locationUri) - ? databaseLocation(databaseId(identifier.namespace())) - : locationUri, - identifier.name()); + Strings.isNullOrEmpty(locationUri) ? databaseLocation(db) : locationUri, identifier.name()); } @Override public List listTables(Namespace namespace) { - ImmutableList dbNames; + ImmutableList dbNames = ImmutableList.of(); if (namespace.isEmpty()) { dbNames = - Streams.stream(bigLakeClient.listDatabases(catalogName)) + Streams.stream(client.listDatabases(catalogName)) .map(db -> DatabaseName.parse(db.getName())) .collect(ImmutableList.toImmutableList()); - } else { + } else if (namespace.levels().length == 1) { dbNames = ImmutableList.of(databaseName(namespace)); + } else { + throwInvalidDbNamespaceError(namespace); } ImmutableList.Builder result = ImmutableList.builder(); dbNames.stream() .map( dbName -> - Streams.stream(bigLakeClient.listTables(dbName)) + Streams.stream(client.listTables(dbName)) .map(BigLakeCatalog::tableIdentifier) .collect(ImmutableList.toImmutableList())) .forEach(result::addAll); @@ -187,6 +183,11 @@ public List listTables(Namespace namespace) { @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { + String db = databaseId(identifier.namespace()); + if (db == null) { + throwInvalidDbNamespaceError(identifier.namespace()); + } + TableOperations ops = null; TableMetadata lastMetadata = null; if (purge) { @@ -196,7 +197,7 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { } try { - bigLakeClient.deleteTable(tableName(databaseId(identifier.namespace()), identifier.name())); + client.deleteTable(tableName(db, identifier.name())); } catch (NoSuchTableException e) { return false; } @@ -210,22 +211,29 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { @Override public void renameTable(TableIdentifier from, TableIdentifier to) { - String fromDbId = databaseId(from.namespace()); - String toDbId = databaseId(to.namespace()); + String fromDb = databaseId(from.namespace()); + if (fromDb == null) { + throwInvalidDbNamespaceError(from.namespace()); + } + + String toDb = databaseId(to.namespace()); + if (toDb == null) { + throwInvalidDbNamespaceError(to.namespace()); + } Preconditions.checkArgument( - fromDbId.equals(toDbId), + fromDb.equals(toDb), "Cannot rename table %s to %s: database must match", from.toString(), to.toString()); - bigLakeClient.renameTable(tableName(fromDbId, from.name()), tableName(toDbId, to.name())); + client.renameTable(tableName(fromDb, from.name()), tableName(toDb, to.name())); } @Override public void createNamespace(Namespace namespace, Map metadata) { if (namespace.isEmpty()) { // Used by `CREATE NAMESPACE `. Create a BLMS catalog linked with Iceberg catalog. - bigLakeClient.createCatalog(catalogName, Catalog.getDefaultInstance()); + client.createCatalog(catalogName, Catalog.getDefaultInstance()); LOG.info("Created BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { // Create a database. @@ -236,8 +244,7 @@ public void createNamespace(Namespace namespace, Map metadata) { .putAllParameters(metadata) .setLocationUri(databaseLocation(dbId)); - bigLakeClient.createDatabase( - DatabaseName.of(bigLakeProjectId, bigLakeRegion, catalogId, dbId), builder.build()); + client.createDatabase(DatabaseName.of(projectId, region, catalogId, dbId), builder.build()); } else { throw new IllegalArgumentException( String.format("Invalid namespace (too long): %s", namespace)); @@ -246,16 +253,18 @@ public void createNamespace(Namespace namespace, Map metadata) { @Override public List listNamespaces(Namespace namespace) { - if (!namespace.isEmpty()) { - // BLMS does not support namespaces under database or tables, returns empty. - // It is called when dropping a namespace to make sure it's empty, returns empty to unblock - // deletion. + if (namespace.isEmpty()) { + return Streams.stream(client.listDatabases(catalogName)) + .map(BigLakeCatalog::namespace) + .collect(ImmutableList.toImmutableList()); + } + + // Database namespace does not have nested namespaces. + if (namespace.levels().length == 1) { return ImmutableList.of(); } - return Streams.stream(bigLakeClient.listDatabases(catalogName)) - .map(BigLakeCatalog::namespace) - .collect(ImmutableList.toImmutableList()); + throw new NoSuchNamespaceException("Invalid namespace: %s", namespace); } @Override @@ -263,10 +272,10 @@ public boolean dropNamespace(Namespace namespace) { try { if (namespace.isEmpty()) { // Used by `DROP NAMESPACE `. Deletes the BLMS catalog linked by Iceberg catalog. - bigLakeClient.deleteCatalog(catalogName); + client.deleteCatalog(catalogName); LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { - bigLakeClient.deleteDatabase(databaseName(namespace)); + client.deleteDatabase(databaseName(namespace)); // Don't delete the data file folder for safety. It aligns with HMS's default behavior. // To support database or catalog level config controlling file deletion in future. } else { @@ -274,6 +283,7 @@ public boolean dropNamespace(Namespace namespace) { return false; } } catch (NoSuchNamespaceException e) { + LOG.warn("Failed to drop namespace", e); return false; } @@ -281,22 +291,30 @@ public boolean dropNamespace(Namespace namespace) { } @Override - public boolean setProperties(Namespace namespace, Map properties) { + public boolean setProperties(Namespace namespace, Map props) { + DatabaseName dbName = databaseName(namespace); + if (dbName == null) { + throwInvalidDbNamespaceError(namespace); + } + HiveDatabaseOptions.Builder optionsBuilder = loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); - properties.forEach(optionsBuilder::putParameters); - bigLakeClient.updateDatabaseParameters( - databaseName(namespace), optionsBuilder.getParametersMap()); + props.forEach(optionsBuilder::putParameters); + client.updateDatabaseParameters(dbName, optionsBuilder.getParametersMap()); return true; } @Override - public boolean removeProperties(Namespace namespace, Set properties) { + public boolean removeProperties(Namespace namespace, Set props) { + DatabaseName dbName = databaseName(namespace); + if (dbName == null) { + throwInvalidDbNamespaceError(namespace); + } + HiveDatabaseOptions.Builder optionsBuilder = loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); - properties.forEach(optionsBuilder::removeParameters); - bigLakeClient.updateDatabaseParameters( - databaseName(namespace), optionsBuilder.getParametersMap()); + props.forEach(optionsBuilder::removeParameters); + client.updateDatabaseParameters(dbName, optionsBuilder.getParametersMap()); return true; } @@ -304,10 +322,10 @@ public boolean removeProperties(Namespace namespace, Set properties) { public Map loadNamespaceMetadata(Namespace namespace) { if (namespace.isEmpty()) { // Calls getCatalog to check existence. BLMS catalog has no metadata today. - bigLakeClient.getCatalog(catalogName); + client.getCatalog(catalogName); return ImmutableMap.of(); } else if (namespace.levels().length == 1) { - return metadata(loadDatabase(namespace)); + return loadDatabase(namespace).getHiveOptions().getParametersMap(); } else { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } @@ -320,7 +338,7 @@ public String name() { @Override protected Map properties() { - return bigLakeProperties == null ? ImmutableMap.of() : bigLakeProperties; + return properties == null ? ImmutableMap.of() : properties; } @Override @@ -342,8 +360,7 @@ protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { private String databaseLocation(String dbId) { String warehouseLocation = - LocationUtil.stripTrailingSlash( - bigLakeProperties.get(CatalogProperties.WAREHOUSE_LOCATION)); + LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set"); return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); } @@ -358,36 +375,24 @@ private static Namespace namespace(Database db) { } private TableName tableName(String dbId, String tableId) { - return TableName.of(bigLakeProjectId, bigLakeRegion, catalogId, dbId, tableId); + return TableName.of(projectId, region, catalogId, dbId, tableId); } private String databaseId(Namespace namespace) { - if (namespace.levels().length != 1) { - throw new NoSuchNamespaceException( - namespace.isEmpty() - ? "Invalid BigLake database namespace: empty" - : String.format( - "BigLake database namespace must use format ., invalid namespace: %s", - namespace)); - } - - return namespace.level(0); + return namespace.levels().length == 1 ? namespace.level(0) : null; } private DatabaseName databaseName(Namespace namespace) { - return DatabaseName.of(bigLakeProjectId, bigLakeRegion, catalogId, databaseId(namespace)); + String db = databaseId(namespace); + return db == null ? null : DatabaseName.of(projectId, region, catalogId, db); } private Database loadDatabase(Namespace namespace) { - return bigLakeClient.getDatabase(databaseName(namespace)); + return client.getDatabase(databaseName(namespace)); } - private static Map metadata(Database db) { - HiveDatabaseOptions options = db.getHiveOptions(); - return new ImmutableMap.Builder() - .putAll(options.getParametersMap()) - // Add the storage location of the database to metadata. - .put("location", options.getLocationUri()) - .build(); + private void throwInvalidDbNamespaceError(Namespace namespace) { + throw new NoSuchNamespaceException( + "Invalid BigLake database namespace: %s", namespace.isEmpty() ? "empty" : namespace); } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 52f9763d1caf..4e7b008e072d 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -46,7 +46,7 @@ public final class BigLakeTableOperations extends BaseMetastoreTableOperations { private final BigLakeClient client; private final FileIO io; - // The name of this Iceberg catalog plugin: spark.sql.catalog.. + // The name of this Iceberg catalog plugin. private final String name; private final TableName tableName; diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 6ed504857906..75faa2f84c44 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -68,7 +68,7 @@ public class BigLakeCatalogTest extends CatalogTests { private BigLakeCatalog bigLakeCatalogUsingMockService; // For tests using a BigLake catalog with a mocked client. - private BigLakeClient mockBigLakeClient = mock(BigLakeClient.class);; + private BigLakeClient mockBigLakeClient = mock(BigLakeClient.class); private BigLakeCatalog bigLakeCatalogUsingMockClient; @BeforeEach @@ -81,8 +81,10 @@ public void before() throws Exception { ImmutableMap properties = ImmutableMap.of( - GCPProperties.BIGLAKE_PROJECT_ID, + GCPProperties.PROJECT_ID, GCP_PROJECT, + GCPProperties.REGION, + GCP_REGION, CatalogProperties.WAREHOUSE_LOCATION, temp.toAbsolutePath().toString()); @@ -94,13 +96,11 @@ public void before() throws Exception { bigLakeCatalogUsingMockService = new BigLakeCatalog(); bigLakeCatalogUsingMockService.setConf(new Configuration()); - bigLakeCatalogUsingMockService.initialize( - CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, new BigLakeClient(settings)); + bigLakeCatalogUsingMockService.initialize(CATALOG_ID, properties, new BigLakeClient(settings)); bigLakeCatalogUsingMockClient = new BigLakeCatalog(); bigLakeCatalogUsingMockClient.setConf(new Configuration()); - bigLakeCatalogUsingMockClient.initialize( - CATALOG_ID, properties, GCP_PROJECT, GCP_REGION, mockBigLakeClient); + bigLakeCatalogUsingMockClient.initialize(CATALOG_ID, properties, mockBigLakeClient); } @AfterEach @@ -247,8 +247,7 @@ public void testSetPropertiesShouldFailWhenNamespacesAreInvalid() { bigLakeCatalogUsingMockClient.setProperties( Namespace.of("db", "tbl"), ImmutableMap.of())) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage( - "BigLake database namespace must use format ., invalid namespace: db.tbl"); + .hasMessage("Invalid BigLake database namespace: db.tbl"); } @Test @@ -286,8 +285,7 @@ public void testRemovePropertiesShouldFailWhenNamespacesAreInvalid() { bigLakeCatalogUsingMockClient.removeProperties( Namespace.of("db", "tbl"), ImmutableSet.of())) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage( - "BigLake database namespace must use format ., invalid namespace: db.tbl"); + .hasMessage("Invalid BigLake database namespace: db.tbl"); } @Test @@ -331,8 +329,7 @@ public void testLoadNamespaceMetadataForDatabases() { .build()); assertThat(bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of("db"))) - .containsAllEntriesOf( - ImmutableMap.of("location", "my location uri", "key1", "value1", "key2", "value2")); + .containsAllEntriesOf(ImmutableMap.of("key1", "value1", "key2", "value2")); } @Test @@ -350,7 +347,6 @@ public void testNewTableOpsShouldfailedForInvalidNamespace() { bigLakeCatalogUsingMockClient.newTableOps( TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage( - "BigLake database namespace must use format ., invalid namespace: n0.n1"); + .hasMessage("Invalid BigLake database namespace: n0.n1"); } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index bb05699836d4..3f06c4153eae 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -84,8 +84,10 @@ public class BigLakeTableOperationsTest { public void before() throws Exception { ImmutableMap properties = ImmutableMap.of( - GCPProperties.BIGLAKE_PROJECT_ID, + GCPProperties.PROJECT_ID, GCP_PROJECT, + GCPProperties.REGION, + GCP_REGION, CatalogProperties.WAREHOUSE_LOCATION, temp.toFile().getAbsolutePath(), GCPProperties.BIGLAKE_CATALOG_ID, @@ -93,7 +95,7 @@ public void before() throws Exception { bigLakeCatalog = new BigLakeCatalog(); bigLakeCatalog.setConf(new Configuration()); - bigLakeCatalog.initialize(CATALOG_NAME, properties, GCP_PROJECT, GCP_REGION, bigLakeClient); + bigLakeCatalog.initialize(CATALOG_NAME, properties, bigLakeClient); tableOps = (BigLakeTableOperations) bigLakeCatalog.newTableOps(TABLE_IDENTIFIER); } From 59d1280a2943ffe5a826456e9c5c94bfe5f3e981 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 18 Jul 2023 01:59:51 +0000 Subject: [PATCH 17/22] fix more review comments --- .../org/apache/iceberg/gcp/biglake/BigLakeCatalog.java | 4 ++-- .../apache/iceberg/gcp/biglake/BigLakeCatalogTest.java | 6 +++--- .../gcp/biglake/BigLakeTableOperationsTest.java | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 94a2017dbc26..e84088380775 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -107,7 +107,7 @@ void initialize(String initName, Map initProperties, BigLakeClie this.name = initName; this.properties = ImmutableMap.copyOf(initProperties); - Preconditions.checkNotNull(initClient, "BigLake client must not be null"); + Preconditions.checkArgument(initClient != null, "BigLake client must not be null"); this.client = initClient; Preconditions.checkArgument( @@ -361,7 +361,7 @@ protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { private String databaseLocation(String dbId) { String warehouseLocation = LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); - Preconditions.checkNotNull(warehouseLocation, "Data warehouse location is not set"); + Preconditions.checkArgument(warehouseLocation != null, "Data warehouse location is not set"); return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 75faa2f84c44..935d5967d7cd 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -236,7 +236,7 @@ public void testDropNamespaceShouldFailWhenNamespaceIsTooLong() { } @Test - public void testSetPropertiesShouldFailWhenNamespacesAreInvalid() { + public void testSetPropertiesShouldFailWhenNamespaceIsInvalid() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.setProperties(Namespace.empty(), ImmutableMap.of())) .isInstanceOf(NoSuchNamespaceException.class) @@ -272,7 +272,7 @@ public void testSetPropertiesShouldSucceedForDatabase() { } @Test - public void testRemovePropertiesShouldFailWhenNamespacesAreInvalid() { + public void testRemovePropertiesShouldFailWhenNamespaceIsInvalid() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.removeProperties( @@ -341,7 +341,7 @@ public void testLoadNamespaceMetadataShouldFailWhenInvalid() { } @Test - public void testNewTableOpsShouldfailedForInvalidNamespace() { + public void testNewTableOpsShouldFailForInvalidNamespace() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.newTableOps( diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 3f06c4153eae..8ac2c8d6d139 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -148,11 +148,12 @@ public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { assertThatThrownBy( () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()) - .isInstanceOf(CommitFailedException.class); + .isInstanceOf(CommitFailedException.class) + .hasMessage("Updating table failed due to conflict updates (etag mismatch)"); } @Test - public void testDoFreshRefreshShouldReturnNullForNonIcebergTable() { + public void testInitRefreshShouldReturnNullForNonIcebergTable() { when(bigLakeClient.getTable(TABLE_NAME)) .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); @@ -176,7 +177,7 @@ private Table createTestTable() throws IOException { .createTransaction() .commitTransaction(); - Optional metadataLocation = getAnyIcebergMetadataFilePath(tableDir); + Optional metadataLocation = getAnyJsonFilePath(tableDir); assertThat(metadataLocation).isPresent(); return Table.newBuilder() .setName(TABLE_NAME.toString()) @@ -187,8 +188,7 @@ private Table createTestTable() throws IOException { .build(); } - private static Optional getAnyIcebergMetadataFilePath(String tableDir) - throws IOException { + private static Optional getAnyJsonFilePath(String tableDir) throws IOException { for (File file : FileUtils.listFiles(new File(tableDir), TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE)) { if (file.getCanonicalPath().endsWith(".json")) { From abc420a833cccd5e2b4e26b2dd808b36a7056b19 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 18 Jul 2023 15:30:31 +0000 Subject: [PATCH 18/22] fix more review comments --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 16 +++++++++++----- .../iceberg/gcp/biglake/BigLakeClient.java | 8 +++++++- .../gcp/biglake/BigLakeTableOperations.java | 10 +++++----- .../iceberg/gcp/biglake/BigLakeCatalogTest.java | 2 +- .../gcp/biglake/BigLakeTableOperationsTest.java | 8 +++++--- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index e84088380775..48061b671d05 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -70,12 +70,14 @@ public final class BigLakeCatalog extends BaseMetastoreCatalog private FileIO io; private Object conf; + private BigLakeClient client; + private String projectId; private String region; // BLMS catalog ID and fully qualified name. private String catalogId; private CatalogName catalogName; - private BigLakeClient client; + private String warehouseLocation; private CloseableGroup closeableGroup; @@ -124,11 +126,18 @@ void initialize(String initName, Map initProperties, BigLakeClie this.catalogId = properties.getOrDefault(GCPProperties.BIGLAKE_CATALOG_ID, initName); this.catalogName = CatalogName.of(projectId, region, catalogId); + Preconditions.checkArgument( + properties.containsKey(CatalogProperties.WAREHOUSE_LOCATION), + "Data warehouse location must be specified"); + this.warehouseLocation = + LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); + String ioImpl = properties.getOrDefault(CatalogProperties.FILE_IO_IMPL, ResolvingFileIO.class.getName()); this.io = CatalogUtil.loadFileIO(ioImpl, properties, conf); this.closeableGroup = new CloseableGroup(); + closeableGroup.addCloseable(client); closeableGroup.addCloseable(io); closeableGroup.setSuppressCloseFailure(true); } @@ -359,10 +368,7 @@ protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { } private String databaseLocation(String dbId) { - String warehouseLocation = - LocationUtil.stripTrailingSlash(properties.get(CatalogProperties.WAREHOUSE_LOCATION)); - Preconditions.checkArgument(warehouseLocation != null, "Data warehouse location is not set"); - return String.format("%s/%s.db", LocationUtil.stripTrailingSlash(warehouseLocation), dbId); + return String.format("%s/%s.db", warehouseLocation, dbId); } private static TableIdentifier tableIdentifier(Table table) { diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index b088c14bd134..82a67e0330f4 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -44,6 +44,7 @@ import com.google.cloud.bigquery.biglake.v1.UpdateTableRequest; import com.google.protobuf.Empty; import com.google.protobuf.FieldMask; +import java.io.Closeable; import java.io.IOException; import java.util.Map; import java.util.function.Supplier; @@ -53,7 +54,7 @@ import org.apache.iceberg.exceptions.NotAuthorizedException; /** A client of Google BigLake service. */ -final class BigLakeClient { +final class BigLakeClient implements Closeable { private final MetastoreServiceClient stub; @@ -277,6 +278,11 @@ public Iterable
listTables(DatabaseName name) { name.getDatabase()); } + @Override + public void close() { + stub.close(); + } + // Converts BigLake API errors to Iceberg errors. private T convertException(Supplier result, String resourceId) { try { diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 4e7b008e072d..29f24b999951 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory; /** Handles BigLake table operations. */ -public final class BigLakeTableOperations extends BaseMetastoreTableOperations { +final class BigLakeTableOperations extends BaseMetastoreTableOperations { private static final Logger LOG = LoggerFactory.getLogger(BigLakeTableOperations.class); @@ -59,7 +59,7 @@ public final class BigLakeTableOperations extends BaseMetastoreTableOperations { // The doRefresh method should provide implementation on how to get the metadata location @Override - public void doRefresh() { + protected void doRefresh() { // Must default to null. String metadataLocation = null; try { @@ -83,7 +83,7 @@ public void doRefresh() { // The doCommit method should provide implementation on how to update with metadata location // atomically @Override - public void doCommit(TableMetadata base, TableMetadata metadata) { + protected void doCommit(TableMetadata base, TableMetadata metadata) { boolean isNewTable = base == null; String newMetadataLocation = writeNewMetadataIfRequired(isNewTable, metadata); @@ -127,7 +127,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } @Override - public String tableName() { + protected String tableName() { return String.format("%s.%s.%s", name, tableName.getDatabase(), tableName.getTable()); } @@ -182,7 +182,7 @@ private void updateTable( } catch (AbortedException e) { if (e.getMessage().toLowerCase().contains("etag mismatch")) { throw new CommitFailedException( - "Updating table failed due to conflict updates (etag mismatch)"); + "Updating table failed due to conflicting updates (etag mismatch)"); } throw e; } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 935d5967d7cd..d04bf5d68b16 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -231,7 +231,7 @@ public void testListTablesShouldListTablesInAllDbsWhenNamespaceIsEmpty() { } @Test - public void testDropNamespaceShouldFailWhenNamespaceIsTooLong() { + public void testDropTooLongNamespace() { assertThat(bigLakeCatalogUsingMockClient.dropNamespace(Namespace.of("n0", "n1"))).isFalse(); } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 8ac2c8d6d139..586ce646f765 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -42,6 +42,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.TableIdentifier; @@ -81,7 +82,7 @@ public class BigLakeTableOperationsTest { private BigLakeTableOperations tableOps; @BeforeEach - public void before() throws Exception { + public void before() { ImmutableMap properties = ImmutableMap.of( GCPProperties.PROJECT_ID, @@ -149,7 +150,7 @@ public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { assertThatThrownBy( () -> loadedTable.updateSchema().addColumn("n", Types.IntegerType.get()).commit()) .isInstanceOf(CommitFailedException.class) - .hasMessage("Updating table failed due to conflict updates (etag mismatch)"); + .hasMessage("Updating table failed due to conflicting updates (etag mismatch)"); } @Test @@ -183,7 +184,8 @@ private Table createTestTable() throws IOException { .setName(TABLE_NAME.toString()) .setHiveOptions( HiveTableOptions.newBuilder() - .putParameters("metadata_location", metadataLocation.get()) + .putParameters( + BaseMetastoreTableOperations.METADATA_LOCATION_PROP, metadataLocation.get()) .setStorageDescriptor(StorageDescriptor.newBuilder().setLocationUri(tableDir))) .build(); } From 28d9dfcf7d50a36805e2eceb753f692784b7c258 Mon Sep 17 00:00:00 2001 From: coufon Date: Wed, 19 Jul 2023 17:14:15 +0000 Subject: [PATCH 19/22] fix more review comment --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 90 +++++++++---------- .../iceberg/gcp/biglake/BigLakeClient.java | 19 ++-- .../gcp/biglake/BigLakeTableOperations.java | 4 +- .../gcp/biglake/BigLakeCatalogTest.java | 22 ++--- .../biglake/BigLakeTableOperationsTest.java | 10 +-- 5 files changed, 69 insertions(+), 76 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index 48061b671d05..bc6dc42b4c36 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -144,25 +144,22 @@ void initialize(String initName, Map initProperties, BigLakeClie @Override protected TableOperations newTableOps(TableIdentifier identifier) { - String db = databaseId(identifier.namespace()); - if (db == null) { - throwInvalidDbNamespaceError(identifier.namespace()); - } + String dbId = databaseId(identifier.namespace()); + validateDatabaseId(dbId, identifier.namespace()); - return new BigLakeTableOperations(client, io, name(), tableName(db, identifier.name())); + return new BigLakeTableOperations(client, io, name(), tableName(dbId, identifier.name())); } @Override protected String defaultWarehouseLocation(TableIdentifier identifier) { - String db = databaseId(identifier.namespace()); - if (db == null) { - throwInvalidDbNamespaceError(identifier.namespace()); - } + String dbId = databaseId(identifier.namespace()); + validateDatabaseId(dbId, identifier.namespace()); - String locationUri = loadDatabase(identifier.namespace()).getHiveOptions().getLocationUri(); + String locationUri = loadDatabase(dbId).getHiveOptions().getLocationUri(); return String.format( "%s/%s", - Strings.isNullOrEmpty(locationUri) ? databaseLocation(db) : locationUri, identifier.name()); + Strings.isNullOrEmpty(locationUri) ? databaseLocation(dbId) : locationUri, + identifier.name()); } @Override @@ -173,10 +170,10 @@ public List listTables(Namespace namespace) { Streams.stream(client.listDatabases(catalogName)) .map(db -> DatabaseName.parse(db.getName())) .collect(ImmutableList.toImmutableList()); - } else if (namespace.levels().length == 1) { - dbNames = ImmutableList.of(databaseName(namespace)); } else { - throwInvalidDbNamespaceError(namespace); + String dbId = databaseId(namespace); + validateDatabaseId(dbId, namespace); + dbNames = ImmutableList.of(databaseName(dbId)); } ImmutableList.Builder result = ImmutableList.builder(); @@ -192,10 +189,8 @@ public List listTables(Namespace namespace) { @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { - String db = databaseId(identifier.namespace()); - if (db == null) { - throwInvalidDbNamespaceError(identifier.namespace()); - } + String dbId = databaseId(identifier.namespace()); + validateDatabaseId(dbId, identifier.namespace()); TableOperations ops = null; TableMetadata lastMetadata = null; @@ -206,7 +201,7 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { } try { - client.deleteTable(tableName(db, identifier.name())); + client.deleteTable(tableName(dbId, identifier.name())); } catch (NoSuchTableException e) { return false; } @@ -221,14 +216,10 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { @Override public void renameTable(TableIdentifier from, TableIdentifier to) { String fromDb = databaseId(from.namespace()); - if (fromDb == null) { - throwInvalidDbNamespaceError(from.namespace()); - } + validateDatabaseId(fromDb, from.namespace()); String toDb = databaseId(to.namespace()); - if (toDb == null) { - throwInvalidDbNamespaceError(to.namespace()); - } + validateDatabaseId(toDb, to.namespace()); Preconditions.checkArgument( fromDb.equals(toDb), @@ -284,7 +275,9 @@ public boolean dropNamespace(Namespace namespace) { client.deleteCatalog(catalogName); LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { - client.deleteDatabase(databaseName(namespace)); + String dbId = databaseId(namespace); + validateDatabaseId(dbId, namespace); + client.deleteDatabase(databaseName(dbId)); // Don't delete the data file folder for safety. It aligns with HMS's default behavior. // To support database or catalog level config controlling file deletion in future. } else { @@ -301,40 +294,38 @@ public boolean dropNamespace(Namespace namespace) { @Override public boolean setProperties(Namespace namespace, Map props) { - DatabaseName dbName = databaseName(namespace); - if (dbName == null) { - throwInvalidDbNamespaceError(namespace); - } + String dbId = databaseId(namespace); + validateDatabaseId(dbId, namespace); HiveDatabaseOptions.Builder optionsBuilder = - loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); + loadDatabase(dbId).toBuilder().getHiveOptionsBuilder(); props.forEach(optionsBuilder::putParameters); - client.updateDatabaseParameters(dbName, optionsBuilder.getParametersMap()); + client.updateDatabaseParameters(databaseName(dbId), optionsBuilder.getParametersMap()); return true; } @Override public boolean removeProperties(Namespace namespace, Set props) { - DatabaseName dbName = databaseName(namespace); - if (dbName == null) { - throwInvalidDbNamespaceError(namespace); - } + String dbId = databaseId(namespace); + validateDatabaseId(dbId, namespace); HiveDatabaseOptions.Builder optionsBuilder = - loadDatabase(namespace).toBuilder().getHiveOptionsBuilder(); + loadDatabase(dbId).toBuilder().getHiveOptionsBuilder(); props.forEach(optionsBuilder::removeParameters); - client.updateDatabaseParameters(dbName, optionsBuilder.getParametersMap()); + client.updateDatabaseParameters(databaseName(dbId), optionsBuilder.getParametersMap()); return true; } @Override public Map loadNamespaceMetadata(Namespace namespace) { if (namespace.isEmpty()) { - // Calls getCatalog to check existence. BLMS catalog has no metadata today. - client.getCatalog(catalogName); + // Calls catalog to check existence. BLMS catalog has no metadata today. + client.catalog(catalogName); return ImmutableMap.of(); } else if (namespace.levels().length == 1) { - return loadDatabase(namespace).getHiveOptions().getParametersMap(); + String dbId = databaseId(namespace); + validateDatabaseId(dbId, namespace); + return loadDatabase(dbId).getHiveOptions().getParametersMap(); } else { throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); } @@ -388,17 +379,18 @@ private String databaseId(Namespace namespace) { return namespace.levels().length == 1 ? namespace.level(0) : null; } - private DatabaseName databaseName(Namespace namespace) { - String db = databaseId(namespace); - return db == null ? null : DatabaseName.of(projectId, region, catalogId, db); + private DatabaseName databaseName(String dbId) { + return DatabaseName.of(projectId, region, catalogId, dbId); } - private Database loadDatabase(Namespace namespace) { - return client.getDatabase(databaseName(namespace)); + private Database loadDatabase(String dbId) { + return client.database(databaseName(dbId)); } - private void throwInvalidDbNamespaceError(Namespace namespace) { - throw new NoSuchNamespaceException( - "Invalid BigLake database namespace: %s", namespace.isEmpty() ? "empty" : namespace); + private void validateDatabaseId(String dbId, Namespace namespace) { + if (dbId == null) { + throw new NoSuchNamespaceException( + "Invalid namespace: %s", namespace.isEmpty() ? "empty" : namespace); + } } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index 82a67e0330f4..2e524b796561 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -88,14 +88,14 @@ public Catalog createCatalog(CatalogName name, Catalog catalog) { name.getCatalog()); } - public Catalog getCatalog(CatalogName name) { + public Catalog catalog(CatalogName name) { return convertException( () -> { try { return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s", name.getCatalog()); + e, "Namespace does not exist: %s (or permission denied)", name.getCatalog()); } }, name.getCatalog()); @@ -109,7 +109,7 @@ public void deleteCatalog(CatalogName name) { return Empty.getDefaultInstance(); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s", name.getCatalog()); + e, "Namespace does not exist: %s (or permission denied)", name.getCatalog()); } }, name.getCatalog()); @@ -129,7 +129,7 @@ public Database createDatabase(DatabaseName name, Database db) { name.getDatabase()); } - public Database getDatabase(DatabaseName name) { + public Database database(DatabaseName name) { return convertException( () -> { try { @@ -137,7 +137,7 @@ public Database getDatabase(DatabaseName name) { GetDatabaseRequest.newBuilder().setName(name.toString()).build()); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s", name.getDatabase()); + e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); } }, name.getDatabase()); @@ -156,7 +156,7 @@ public Database updateDatabaseParameters(DatabaseName name, Map .build()); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s", name.getDatabase()); + e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); } }, name.getDatabase()); @@ -179,7 +179,7 @@ public void deleteDatabase(DatabaseName name) { return Empty.getDefaultInstance(); } catch (PermissionDeniedException e) { throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s", name.getDatabase()); + e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); } }, name.getDatabase()); @@ -202,7 +202,7 @@ public Table createTable(TableName name, Table table) { name.getTable()); } - public Table getTable(TableName name) { + public Table table(TableName name) { if (name.getTable().isEmpty()) { throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); } @@ -288,8 +288,9 @@ private T convertException(Supplier result, String resourceId) { try { return result.get(); } catch (PermissionDeniedException e) { - throw new NotAuthorizedException(e, "BigLake API permission denied"); + throw new NotAuthorizedException(e, "Permission denied"); } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + // "Table already exists" error should be caught earlier. throw new AlreadyExistsException(e, "Namespace already exists: %s", resourceId); } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java index 29f24b999951..d734758f3d93 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperations.java @@ -63,7 +63,7 @@ protected void doRefresh() { // Must default to null. String metadataLocation = null; try { - HiveTableOptions hiveOptions = client.getTable(tableName).getHiveOptions(); + HiveTableOptions hiveOptions = client.table(tableName).getHiveOptions(); if (!hiveOptions.containsParameters(METADATA_LOCATION_PROP)) { throw new NoSuchIcebergTableException( "Invalid Iceberg table %s: missing metadata location", tableName()); @@ -144,7 +144,7 @@ private void createTable(String newMetadataLocation, TableMetadata metadata) { /** Update table properties with concurrent update detection using etag. */ private void updateTable( String oldMetadataLocation, String newMetadataLocation, TableMetadata metadata) { - Table table = client.getTable(tableName); + Table table = client.table(tableName); String etag = table.getEtag(); Preconditions.checkArgument( !etag.isEmpty(), diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index d04bf5d68b16..4575722fba3f 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -135,7 +135,7 @@ protected boolean supportsNamesWithSlashes() { @Test public void testDefaultWarehouseWithDatabaseLocation() { - when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.database(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions(HiveDatabaseOptions.newBuilder().setLocationUri("db_folder")) @@ -149,7 +149,7 @@ public void testDefaultWarehouseWithDatabaseLocation() { @Test public void testDefaultWarehouseWithoutDatabaseLocation() { - when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) + when(mockBigLakeClient.database(DatabaseName.of(GCP_PROJECT, "us", CATALOG_ID, "db"))) .thenReturn( Database.newBuilder().setHiveOptions(HiveDatabaseOptions.getDefaultInstance()).build()); @@ -240,19 +240,19 @@ public void testSetPropertiesShouldFailWhenNamespaceIsInvalid() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.setProperties(Namespace.empty(), ImmutableMap.of())) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Invalid BigLake database namespace: empty"); + .hasMessage("Invalid namespace: empty"); assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.setProperties( Namespace.of("db", "tbl"), ImmutableMap.of())) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Invalid BigLake database namespace: db.tbl"); + .hasMessage("Invalid namespace: db.tbl"); } @Test public void testSetPropertiesShouldSucceedForDatabase() { - when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.database(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions( @@ -278,19 +278,19 @@ public void testRemovePropertiesShouldFailWhenNamespaceIsInvalid() { bigLakeCatalogUsingMockClient.removeProperties( Namespace.empty(), ImmutableSet.of())) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Invalid BigLake database namespace: empty"); + .hasMessage("Invalid namespace: empty"); assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.removeProperties( Namespace.of("db", "tbl"), ImmutableSet.of())) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Invalid BigLake database namespace: db.tbl"); + .hasMessage("Invalid namespace: db.tbl"); } @Test public void testRemovePropertiesShouldSucceedForDatabase() { - when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.database(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions( @@ -313,12 +313,12 @@ public void testRemovePropertiesShouldSucceedForDatabase() { public void testEmptyNamespaceLoadsCatalogMetadata() { assertThat(bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.empty())).isEmpty(); verify(mockBigLakeClient, times(1)) - .getCatalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); + .catalog(CatalogName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID)); } @Test public void testLoadNamespaceMetadataForDatabases() { - when(mockBigLakeClient.getDatabase(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) + when(mockBigLakeClient.database(DatabaseName.of(GCP_PROJECT, GCP_REGION, CATALOG_ID, "db"))) .thenReturn( Database.newBuilder() .setHiveOptions( @@ -347,6 +347,6 @@ public void testNewTableOpsShouldFailForInvalidNamespace() { bigLakeCatalogUsingMockClient.newTableOps( TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Invalid BigLake database namespace: n0.n1"); + .hasMessage("Invalid namespace: n0.n1"); } } diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java index 586ce646f765..0339903f4204 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeTableOperationsTest.java @@ -107,13 +107,13 @@ public void after() throws Exception { @Test public void testDoCommitShouldUseEtagForUpdateTable() throws Exception { - when(bigLakeClient.getTable(TABLE_NAME)) + when(bigLakeClient.table(TABLE_NAME)) .thenThrow(new NoSuchTableException("error message getTable")); Table createdTable = createTestTable(); Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); reset(bigLakeClient); - when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); + when(bigLakeClient.table(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(TABLE_IDENTIFIER); @@ -130,13 +130,13 @@ public void testDoCommitShouldUseEtagForUpdateTable() throws Exception { @Test public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { - when(bigLakeClient.getTable(TABLE_NAME)) + when(bigLakeClient.table(TABLE_NAME)) .thenThrow(new NoSuchTableException("error message getTable")); Table createdTable = createTestTable(); Table tableWithEtag = createdTable.toBuilder().setEtag("etag").build(); reset(bigLakeClient); - when(bigLakeClient.getTable(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); + when(bigLakeClient.table(TABLE_NAME)).thenReturn(tableWithEtag, tableWithEtag); org.apache.iceberg.Table loadedTable = bigLakeCatalog.loadTable(TABLE_IDENTIFIER); @@ -155,7 +155,7 @@ public void testDoCommitShouldFailWhenEtagMismatch() throws Exception { @Test public void testInitRefreshShouldReturnNullForNonIcebergTable() { - when(bigLakeClient.getTable(TABLE_NAME)) + when(bigLakeClient.table(TABLE_NAME)) .thenReturn(Table.newBuilder().setName(TABLE_NAME.toString()).build()); assertThat(tableOps.refresh()).isNull(); From 23c1dc2c3e58c6a7e936dbccb42702120c279d66 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 25 Jul 2023 02:07:52 +0000 Subject: [PATCH 20/22] fix review comments --- .../iceberg/gcp/biglake/BigLakeClient.java | 203 ++++++++---------- 1 file changed, 84 insertions(+), 119 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index 2e524b796561..90c66d1b4475 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -42,7 +42,6 @@ import com.google.cloud.bigquery.biglake.v1.TableName; import com.google.cloud.bigquery.biglake.v1.UpdateDatabaseRequest; import com.google.cloud.bigquery.biglake.v1.UpdateTableRequest; -import com.google.protobuf.Empty; import com.google.protobuf.FieldMask; import java.io.Closeable; import java.io.IOException; @@ -78,88 +77,80 @@ final class BigLakeClient implements Closeable { public Catalog createCatalog(CatalogName name, Catalog catalog) { return convertException( - () -> - stub.createCatalog( + () -> { + try { + return stub.createCatalog( CreateCatalogRequest.newBuilder() .setParent(LocationName.of(name.getProject(), name.getLocation()).toString()) .setCatalogId(name.getCatalog()) .setCatalog(catalog) - .build()), + .build()); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "Catalog already exists: %s", name.getCatalog()); + } + }, name.getCatalog()); } public Catalog catalog(CatalogName name) { - return convertException( - () -> { - try { - return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s (or permission denied)", name.getCatalog()); - } - }, - name.getCatalog()); + try { + return stub.getCatalog(GetCatalogRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Catalog does not exist: %s (or permission denied)", name.getCatalog()); + } } public void deleteCatalog(CatalogName name) { - convertException( - () -> { - try { - stub.deleteCatalog(DeleteCatalogRequest.newBuilder().setName(name.toString()).build()); - return Empty.getDefaultInstance(); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s (or permission denied)", name.getCatalog()); - } - }, - name.getCatalog()); + try { + stub.deleteCatalog(DeleteCatalogRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Catalog does not exist: %s (or permission denied)", name.getCatalog()); + } } public Database createDatabase(DatabaseName name, Database db) { return convertException( - () -> - stub.createDatabase( + () -> { + try { + return stub.createDatabase( CreateDatabaseRequest.newBuilder() .setParent( CatalogName.of(name.getProject(), name.getLocation(), name.getCatalog()) .toString()) .setDatabaseId(name.getDatabase()) .setDatabase(db) - .build()), + .build()); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "Namespace already exists: %s", name.getDatabase()); + } + }, name.getDatabase()); } public Database database(DatabaseName name) { - return convertException( - () -> { - try { - return stub.getDatabase( - GetDatabaseRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); - } - }, - name.getDatabase()); + try { + return stub.getDatabase(GetDatabaseRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); + } } public Database updateDatabaseParameters(DatabaseName name, Map parameters) { Database.Builder builder = Database.newBuilder().setName(name.toString()); builder.getHiveOptionsBuilder().putAllParameters(parameters); - return convertException( - () -> { - try { - return stub.updateDatabase( - UpdateDatabaseRequest.newBuilder() - .setDatabase(builder) - .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) - .build()); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); - } - }, - name.getDatabase()); + try { + return stub.updateDatabase( + UpdateDatabaseRequest.newBuilder() + .setDatabase(builder) + .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); + } } public Iterable listDatabases(CatalogName name) { @@ -171,18 +162,12 @@ public Iterable listDatabases(CatalogName name) { } public void deleteDatabase(DatabaseName name) { - convertException( - () -> { - try { - stub.deleteDatabase( - DeleteDatabaseRequest.newBuilder().setName(name.toString()).build()); - return Empty.getDefaultInstance(); - } catch (PermissionDeniedException e) { - throw new NoSuchNamespaceException( - e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); - } - }, - name.getDatabase()); + try { + stub.deleteDatabase(DeleteDatabaseRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchNamespaceException( + e, "Namespace does not exist: %s (or permission denied)", name.getDatabase()); + } } public Table createTable(TableName name, Table table) { @@ -206,68 +191,51 @@ public Table table(TableName name) { if (name.getTable().isEmpty()) { throw new NoSuchTableException("BigLake API does not allow tables with empty ID"); } - return convertException( - () -> { - try { - return stub.getTable(GetTableRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table does not exist: %s (or permission denied)", name.getTable()); - } - }, - name.getTable()); + try { + return stub.getTable(GetTableRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } } public Table updateTableParameters(TableName name, Map parameters, String etag) { Table.Builder builder = Table.newBuilder().setName(name.toString()).setEtag(etag); builder.getHiveOptionsBuilder().putAllParameters(parameters); - return convertException( - () -> { - try { - return stub.updateTable( - UpdateTableRequest.newBuilder() - .setTable(builder) - .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) - .build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table does not exist: %s (or permission denied)", name.getTable()); - } - }, - name.getTable()); + try { + return stub.updateTable( + UpdateTableRequest.newBuilder() + .setTable(builder) + .setUpdateMask(FieldMask.newBuilder().addPaths("hive_options.parameters")) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } } public Table renameTable(TableName name, TableName newName) { - return convertException( - () -> { - try { - return stub.renameTable( - RenameTableRequest.newBuilder() - .setName(name.toString()) - .setNewName(newName.toString()) - .build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table does not exist: %s (or permission denied)", name.getTable()); - } catch (com.google.api.gax.rpc.AlreadyExistsException e) { - throw new AlreadyExistsException(e, "Table already exists: %s", newName.getTable()); - } - }, - name.getTable()); + try { + return stub.renameTable( + RenameTableRequest.newBuilder() + .setName(name.toString()) + .setNewName(newName.toString()) + .build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } catch (com.google.api.gax.rpc.AlreadyExistsException e) { + throw new AlreadyExistsException(e, "Table already exists: %s", newName.getTable()); + } } public Table deleteTable(TableName name) { - return convertException( - () -> { - try { - return stub.deleteTable( - DeleteTableRequest.newBuilder().setName(name.toString()).build()); - } catch (PermissionDeniedException e) { - throw new NoSuchTableException( - e, "Table does not exist: %s (or permission denied)", name.getTable()); - } - }, - name.getTable()); + try { + return stub.deleteTable(DeleteTableRequest.newBuilder().setName(name.toString()).build()); + } catch (PermissionDeniedException e) { + throw new NoSuchTableException( + e, "Table does not exist: %s (or permission denied)", name.getTable()); + } } public Iterable
listTables(DatabaseName name) { @@ -289,9 +257,6 @@ private T convertException(Supplier result, String resourceId) { return result.get(); } catch (PermissionDeniedException e) { throw new NotAuthorizedException(e, "Permission denied"); - } catch (com.google.api.gax.rpc.AlreadyExistsException e) { - // "Table already exists" error should be caught earlier. - throw new AlreadyExistsException(e, "Namespace already exists: %s", resourceId); } } From afcdd6e239a8c56c35135e8f05238c772cb17820 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 3 Oct 2023 18:34:04 +0000 Subject: [PATCH 21/22] fix review comment --- .../iceberg/gcp/biglake/BigLakeCatalog.java | 144 ++++++++++-------- .../iceberg/gcp/biglake/BigLakeClient.java | 9 +- .../gcp/biglake/BigLakeCatalogTest.java | 7 +- 3 files changed, 93 insertions(+), 67 deletions(-) diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java index bc6dc42b4c36..e907e2d38354 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeCatalog.java @@ -40,6 +40,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.ServiceFailureException; import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.hadoop.Configurable; @@ -98,7 +99,7 @@ public void initialize(String initName, Map initProperties) { new BigLakeClient( initProperties.getOrDefault(CatalogProperties.URI, DEFAULT_BIGLAKE_SERVICE_ENDPOINT)); } catch (IOException e) { - throw new ServiceFailureException(e, "Creating BigLake client failed"); + throw new ServiceFailureException(e, "Failed to create BigLake client"); } initialize(initName, initProperties, initClient); @@ -144,65 +145,78 @@ void initialize(String initName, Map initProperties, BigLakeClie @Override protected TableOperations newTableOps(TableIdentifier identifier) { - String dbId = databaseId(identifier.namespace()); - validateDatabaseId(dbId, identifier.namespace()); + if (!isValidIdentifier(identifier)) { + throw new NoSuchTableException("Invalid identifier: %s", identifier); + } + String dbId = databaseId(identifier.namespace()); return new BigLakeTableOperations(client, io, name(), tableName(dbId, identifier.name())); } @Override protected String defaultWarehouseLocation(TableIdentifier identifier) { - String dbId = databaseId(identifier.namespace()); - validateDatabaseId(dbId, identifier.namespace()); + if (!isValidIdentifier(identifier)) { + throw new NoSuchTableException("Invalid identifier: %s", identifier); + } + String dbId = databaseId(identifier.namespace()); String locationUri = loadDatabase(dbId).getHiveOptions().getLocationUri(); return String.format( "%s/%s", - Strings.isNullOrEmpty(locationUri) ? databaseLocation(dbId) : locationUri, - identifier.name()); + Strings.isNullOrEmpty(locationUri) ? newDbLocation(dbId) : locationUri, identifier.name()); } @Override public List listTables(Namespace namespace) { - ImmutableList dbNames = ImmutableList.of(); + ImmutableList dbNames; if (namespace.isEmpty()) { dbNames = Streams.stream(client.listDatabases(catalogName)) .map(db -> DatabaseName.parse(db.getName())) .collect(ImmutableList.toImmutableList()); } else { - String dbId = databaseId(namespace); - validateDatabaseId(dbId, namespace); - dbNames = ImmutableList.of(databaseName(dbId)); + if (namespace.levels().length != 1) { + throw new NoSuchNamespaceException( + "Invalid namespace: %s", namespace.isEmpty() ? "empty" : namespace); + } + + dbNames = ImmutableList.of(databaseName(databaseId(namespace))); } ImmutableList.Builder result = ImmutableList.builder(); - dbNames.stream() - .map( + return dbNames.stream() + .flatMap( dbName -> - Streams.stream(client.listTables(dbName)) - .map(BigLakeCatalog::tableIdentifier) - .collect(ImmutableList.toImmutableList())) - .forEach(result::addAll); - return result.build(); + Streams.stream(client.listTables(dbName)).map(BigLakeCatalog::tableIdentifier)) + .collect(ImmutableList.toImmutableList()); } @Override public boolean dropTable(TableIdentifier identifier, boolean purge) { - String dbId = databaseId(identifier.namespace()); - validateDatabaseId(dbId, identifier.namespace()); + if (!isValidIdentifier(identifier)) { + throw new NoSuchTableException("Invalid identifier: %s", identifier); + } TableOperations ops = null; TableMetadata lastMetadata = null; if (purge) { ops = newTableOps(identifier); - // TODO: to catch NotFoundException as in https://github.com/apache/iceberg/pull/5510. - lastMetadata = ops.current(); + + try { + lastMetadata = ops.current(); + } catch (NotFoundException e) { + LOG.warn( + "Failed to load table metadata for table: {}, continuing drop without purge", + identifier, + e); + } } try { + String dbId = databaseId(identifier.namespace()); client.deleteTable(tableName(dbId, identifier.name())); } catch (NoSuchTableException e) { + LOG.warn("Table not exist or permission denied", e); return false; } @@ -215,22 +229,32 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) { @Override public void renameTable(TableIdentifier from, TableIdentifier to) { - String fromDb = databaseId(from.namespace()); - validateDatabaseId(fromDb, from.namespace()); + if (!isValidIdentifier(from)) { + throw new NoSuchTableException("Invalid identifier: %s", from); + } - String toDb = databaseId(to.namespace()); - validateDatabaseId(toDb, to.namespace()); + if (!isValidIdentifier(to)) { + throw new NoSuchTableException("Invalid identifier: %s", to); + } + String fromDb = databaseId(from.namespace()); + String toDb = databaseId(to.namespace()); Preconditions.checkArgument( fromDb.equals(toDb), "Cannot rename table %s to %s: database must match", from.toString(), to.toString()); + client.renameTable(tableName(fromDb, from.name()), tableName(toDb, to.name())); } @Override public void createNamespace(Namespace namespace, Map metadata) { + if (namespace.levels().length > 1) { + throw new IllegalArgumentException( + String.format("Invalid namespace (too long): %s", namespace)); + } + if (namespace.isEmpty()) { // Used by `CREATE NAMESPACE `. Create a BLMS catalog linked with Iceberg catalog. client.createCatalog(catalogName, Catalog.getDefaultInstance()); @@ -242,21 +266,16 @@ public void createNamespace(Namespace namespace, Map metadata) { builder .getHiveOptionsBuilder() .putAllParameters(metadata) - .setLocationUri(databaseLocation(dbId)); + .setLocationUri(newDbLocation(dbId)); client.createDatabase(DatabaseName.of(projectId, region, catalogId, dbId), builder.build()); - } else { - throw new IllegalArgumentException( - String.format("Invalid namespace (too long): %s", namespace)); } } @Override public List listNamespaces(Namespace namespace) { - if (namespace.isEmpty()) { - return Streams.stream(client.listDatabases(catalogName)) - .map(BigLakeCatalog::namespace) - .collect(ImmutableList.toImmutableList()); + if (namespace.levels().length > 1) { + throw new NoSuchNamespaceException("Invalid namespace (too long): %s", namespace); } // Database namespace does not have nested namespaces. @@ -264,7 +283,9 @@ public List listNamespaces(Namespace namespace) { return ImmutableList.of(); } - throw new NoSuchNamespaceException("Invalid namespace: %s", namespace); + return Streams.stream(client.listDatabases(catalogName)) + .map(BigLakeCatalog::namespace) + .collect(ImmutableList.toImmutableList()); } @Override @@ -275,17 +296,14 @@ public boolean dropNamespace(Namespace namespace) { client.deleteCatalog(catalogName); LOG.info("Deleted BigLake catalog: {}", catalogName.toString()); } else if (namespace.levels().length == 1) { - String dbId = databaseId(namespace); - validateDatabaseId(dbId, namespace); - client.deleteDatabase(databaseName(dbId)); + client.deleteDatabase(databaseName(databaseId(namespace))); // Don't delete the data file folder for safety. It aligns with HMS's default behavior. // To support database or catalog level config controlling file deletion in future. } else { - LOG.warn("Invalid namespace (too long): {}", namespace); return false; } } catch (NoSuchNamespaceException e) { - LOG.warn("Failed to drop namespace", e); + LOG.warn("Namespace not exist or permission denied", e); return false; } @@ -294,9 +312,12 @@ public boolean dropNamespace(Namespace namespace) { @Override public boolean setProperties(Namespace namespace, Map props) { - String dbId = databaseId(namespace); - validateDatabaseId(dbId, namespace); + if (namespace.levels().length != 1) { + throw new NoSuchNamespaceException( + "Invalid namespace: %s", namespace.isEmpty() ? "empty" : namespace); + } + String dbId = databaseId(namespace); HiveDatabaseOptions.Builder optionsBuilder = loadDatabase(dbId).toBuilder().getHiveOptionsBuilder(); props.forEach(optionsBuilder::putParameters); @@ -306,9 +327,12 @@ public boolean setProperties(Namespace namespace, Map props) { @Override public boolean removeProperties(Namespace namespace, Set props) { - String dbId = databaseId(namespace); - validateDatabaseId(dbId, namespace); + if (namespace.levels().length != 1) { + throw new NoSuchNamespaceException( + "Invalid namespace: %s", namespace.isEmpty() ? "empty" : namespace); + } + String dbId = databaseId(namespace); HiveDatabaseOptions.Builder optionsBuilder = loadDatabase(dbId).toBuilder().getHiveOptionsBuilder(); props.forEach(optionsBuilder::removeParameters); @@ -318,17 +342,17 @@ public boolean removeProperties(Namespace namespace, Set props) { @Override public Map loadNamespaceMetadata(Namespace namespace) { - if (namespace.isEmpty()) { - // Calls catalog to check existence. BLMS catalog has no metadata today. - client.catalog(catalogName); - return ImmutableMap.of(); - } else if (namespace.levels().length == 1) { - String dbId = databaseId(namespace); - validateDatabaseId(dbId, namespace); - return loadDatabase(dbId).getHiveOptions().getParametersMap(); - } else { - throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); + if (namespace.levels().length > 1) { + throw new NoSuchNamespaceException("Invalid namespace (too long): %s", namespace); + } + + if (namespace.levels().length == 1) { + return loadDatabase(databaseId(namespace)).getHiveOptions().getParametersMap(); } + + // Calls catalog to check existence. BLMS catalog has no metadata today. + client.catalog(catalogName); + return ImmutableMap.of(); } @Override @@ -358,7 +382,8 @@ protected boolean isValidIdentifier(TableIdentifier tableIdentifier) { return tableIdentifier.namespace().levels().length == 1; } - private String databaseLocation(String dbId) { + /** Returns a new location path of a database, used when it is not already known. */ + private String newDbLocation(String dbId) { return String.format("%s/%s.db", warehouseLocation, dbId); } @@ -375,7 +400,7 @@ private TableName tableName(String dbId, String tableId) { return TableName.of(projectId, region, catalogId, dbId, tableId); } - private String databaseId(Namespace namespace) { + private static String databaseId(Namespace namespace) { return namespace.levels().length == 1 ? namespace.level(0) : null; } @@ -386,11 +411,4 @@ private DatabaseName databaseName(String dbId) { private Database loadDatabase(String dbId) { return client.database(databaseName(dbId)); } - - private void validateDatabaseId(String dbId, Namespace namespace) { - if (dbId == null) { - throw new NoSuchNamespaceException( - "Invalid namespace: %s", namespace.isEmpty() ? "empty" : namespace); - } - } } diff --git a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java index 90c66d1b4475..bcbfde481386 100644 --- a/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java +++ b/gcp/src/main/java/org/apache/iceberg/gcp/biglake/BigLakeClient.java @@ -52,7 +52,12 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NotAuthorizedException; -/** A client of Google BigLake service. */ +/** + * A client of Google BigLake service. + * + *

This client returns 403 (permission denied) for both 403 and 404 (not found). Therefore + * NoSuchTableException and NoSuchNamespaceException could be caused by permission denied. + */ final class BigLakeClient implements Closeable { private final MetastoreServiceClient stub; @@ -173,6 +178,8 @@ public void deleteDatabase(DatabaseName name) { public Table createTable(TableName name, Table table) { return convertException( () -> { + // TODO: to capture 403 errors and check error message to determine whether it fails due + // to parent not found or permission denied, and return proper Iceberg errors. try { return stub.createTable( CreateTableRequest.newBuilder() diff --git a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java index 4575722fba3f..36398abbdc1f 100644 --- a/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java +++ b/gcp/src/test/java/org/apache/iceberg/gcp/biglake/BigLakeCatalogTest.java @@ -46,6 +46,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.gcp.GCPProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -337,7 +338,7 @@ public void testLoadNamespaceMetadataShouldFailWhenInvalid() { assertThatThrownBy( () -> bigLakeCatalogUsingMockClient.loadNamespaceMetadata(Namespace.of("n0", "n1"))) .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Namespace does not exist: n0.n1"); + .hasMessage("Invalid namespace (too long): n0.n1"); } @Test @@ -346,7 +347,7 @@ public void testNewTableOpsShouldFailForInvalidNamespace() { () -> bigLakeCatalogUsingMockClient.newTableOps( TableIdentifier.of(Namespace.of("n0", "n1"), "tbl"))) - .isInstanceOf(NoSuchNamespaceException.class) - .hasMessage("Invalid namespace: n0.n1"); + .isInstanceOf(NoSuchTableException.class) + .hasMessage("Invalid identifier: n0.n1.tbl"); } } From 469b09e2152e3bdb26616f8a142a37bb55ebc4b0 Mon Sep 17 00:00:00 2001 From: coufon Date: Tue, 3 Oct 2023 18:37:19 +0000 Subject: [PATCH 22/22] fix an extra space --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index dd8ff180d28c..6dc876c1c0c5 100644 --- a/build.gradle +++ b/build.gradle @@ -637,7 +637,7 @@ project(':iceberg-gcp') { testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts') - + testImplementation libs.esotericsoftware.kryo testImplementation libs.google.cloud.biglake.grpc testImplementation libs.google.cloud.nio