GH-54902: [C++] Move GCS and google-cloud-cpp to its own libarrow_gcs.so

ci/scripts/python_wheel_xlinux_build.sh

@@ -20,7 +20,16 @@
 set -ex
 
 function check_arrow_visibility {
-    nm --demangle --dynamic /tmp/arrow-dist/lib/libarrow.so > nm_arrow.log
+    local libs=(libarrow.so)
+    if [[ "${ARROW_S3}" == "ON" ]]; then
+        libs+=(libarrow_s3.so)
+    fi
+    if [[ "${ARROW_GCS}" == "ON" ]]; then
+        libs+=(libarrow_gcs.so)
+    fi
+    for lib in "${libs[@]}"; do
+        nm --demangle --dynamic "/tmp/arrow-dist/lib/${lib}"
+    done > nm_arrow.log
 
     # Filter out Arrow symbols and see if anything remains.
     # '_init' and '_fini' symbols may or not be present, we don't care.
@@ -37,7 +46,7 @@ function check_arrow_visibility {
     if [[ -f visible_symbols.log && ! -s visible_symbols.log ]]; then
         return 0
     else
-        echo "== Unexpected symbols exported by libarrow.so =="
+        echo "== Unexpected symbols exported by ${libs[*]} =="
         cat visible_symbols.log
         echo "================================================"
 

cpp/CMakeLists.txt

@@ -484,6 +484,15 @@ if(ARROW_BUILD_STATIC)
 endif()
 set(ARROW_FLIGHT_PC_REQUIRES_PRIVATE "")
 
+# For arrow-gcs.pc.
+set(ARROW_GCS_PC_CFLAGS "")
+set(ARROW_GCS_PC_CFLAGS_PRIVATE "")
+if(ARROW_BUILD_STATIC)
+  string(APPEND ARROW_GCS_PC_CFLAGS_PRIVATE " -DARROW_GCS_STATIC")
+endif()
+set(ARROW_GCS_PC_LIBS_PRIVATE "")
+set(ARROW_GCS_PC_REQUIRES_PRIVATE "")
+
 # For arrow-s3.pc.
 set(ARROW_S3_PC_CFLAGS "")
 set(ARROW_S3_PC_CFLAGS_PRIVATE "")

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1434,7 +1434,11 @@ endmacro()
 # pkg-config
 
 # SDK libraries (on macOS) may be available without .pc files
-macro(arrow_append_pc_system_library PC_PACKAGE PC_PREFIX FALLBACK)
+macro(arrow_append_pc_system_library
+      PC_PACKAGE
+      PC_PREFIX
+      FALLBACK
+      ARROW_PC_TARGET)
   find_package(PkgConfig QUIET)
   if(PkgConfig_FOUND)
     pkg_check_modules(${PC_PREFIX}
@@ -1444,10 +1448,10 @@ macro(arrow_append_pc_system_library PC_PACKAGE PC_PREFIX FALLBACK)
                       QUIET)
   endif()
   if(PkgConfig_FOUND AND ${PC_PREFIX}_FOUND)
-    string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${PC_PACKAGE}")
+    string(APPEND ${ARROW_PC_TARGET}_PC_REQUIRES_PRIVATE " ${PC_PACKAGE}")
   else()
-    message(STATUS "No .pc for ${PC_PACKAGE}. Using ${FALLBACK} in arrow.pc")
-    string(APPEND ARROW_PC_LIBS_PRIVATE " ${FALLBACK}")
+    message(STATUS "No .pc for ${PC_PACKAGE}. Using ${FALLBACK} for ${ARROW_PC_TARGET}")
+    string(APPEND ${ARROW_PC_TARGET}_PC_LIBS_PRIVATE " ${FALLBACK}")
   endif()
 endmacro()
 
@@ -3876,8 +3880,14 @@ if(ARROW_WITH_GOOGLE_CLOUD_CPP)
 
   # curl is required on all platforms. We always use system curl to
   # avoid conflict.
-  find_curl(ARROW)
-  resolve_dependency(google_cloud_cpp_storage PC_PACKAGE_NAMES google_cloud_cpp_storage)
+  find_curl(ARROW_GCS)
+  resolve_dependency(google_cloud_cpp_storage
+                     ARROW_CMAKE_PACKAGE_NAME
+                     ArrowGcs
+                     ARROW_PC_PACKAGE_NAME
+                     arrow-gcs
+                     PC_PACKAGE_NAMES
+                     google_cloud_cpp_storage)
   get_target_property(google_cloud_cpp_storage_INCLUDE_DIR google-cloud-cpp::storage
                       INTERFACE_INCLUDE_DIRECTORIES)
   message(STATUS "Found google-cloud-cpp::storage headers: ${google_cloud_cpp_storage_INCLUDE_DIR}"
@@ -4432,16 +4442,18 @@ if(ARROW_WITH_AZURE_SDK)
 endif()
 
 if(ARROW_BUILD_STATIC)
-  if((ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
-     OR (ARROW_AZURE
-         AND AZURE_SDK_VENDORED
-         AND NOT WIN32))
-    arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl")
+  if(ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
+    arrow_append_pc_system_library("libcurl" ARROW_GCS_CURL_PC "-lcurl" ARROW_GCS)
+  endif()
+  if(ARROW_AZURE
+     AND AZURE_SDK_VENDORED
+     AND NOT WIN32)
+    arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl" ARROW)
   endif()
   if(ARROW_AZURE
      AND AZURE_SDK_VENDORED
      AND NOT WIN32)
-    arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2")
+    arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2" ARROW)
   endif()
   if(ARROW_GCS
      AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED"
@@ -4467,14 +4479,14 @@ if(ARROW_BUILD_STATIC)
                           NO_CMAKE_ENVIRONMENT_PATH
                           QUIET)
         if(ARROW_GCS_${ARROW_GCS_ABSL_PC_PACKAGE}_PC_FOUND)
-          string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${ARROW_GCS_ABSL_PC_PACKAGE}")
+          string(APPEND ARROW_GCS_PC_REQUIRES_PRIVATE " ${ARROW_GCS_ABSL_PC_PACKAGE}")
         else()
           message(STATUS "No .pc for ${ARROW_GCS_ABSL_PC_PACKAGE}; "
                          "static pkg-config metadata may be incomplete. Consider CMake")
         endif()
       endforeach()
     else()
-      message(STATUS "PkgConfig not available. Skipping Abseil dependencies from arrow.pc"
+      message(STATUS "PkgConfig not available. Skipping Abseil dependencies from arrow-gcs.pc"
       )
     endif()
   endif()

cpp/src/arrow/ArrowGcsConfig.cmake.in

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This config sets the following variables in your project::
+#
+#   ArrowGcs_FOUND - true if Arrow GCS found on the system
+#
+# This config sets the following targets in your project::
+#
+#   ArrowGcs::arrow_gcs_shared - for linked as shared library if shared library is built
+#   ArrowGcs::arrow_gcs_static - for linked as static library if static library is built
+
+@PACKAGE_INIT@
+
+set(ARROW_GCS_SYSTEM_DEPENDENCIES "@ARROW_GCS_SYSTEM_DEPENDENCIES@")
+
+include(CMakeFindDependencyMacro)
+find_dependency(Arrow CONFIG)
+
+if(ARROW_BUILD_STATIC)
+  arrow_find_dependencies("${ARROW_GCS_SYSTEM_DEPENDENCIES}")
+endif()
+
+include("${CMAKE_CURRENT_LIST_DIR}/ArrowGcsTargets.cmake")
+
+arrow_keep_backward_compatibility(ArrowGcs arrow_gcs)
+
+check_required_components(ArrowGcs)
+
+arrow_show_details(ArrowGcs ARROW_GCS)

cpp/src/arrow/CMakeLists.txt

@@ -33,29 +33,6 @@ set(ARROW_SHARED_PRIVATE_LINK_LIBS)
 set(ARROW_SHARED_INSTALL_INTERFACE_LIBS)
 set(ARROW_STATIC_INSTALL_INTERFACE_LIBS)
 
-if(ARROW_GCS)
-  if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
-    list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
-  elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
-    list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
-    if(absl_SOURCE STREQUAL "SYSTEM")
-      # Bundled google-cloud-cpp with system Abseil needs these for static consumers.
-      list(APPEND
-           ARROW_STATIC_INSTALL_INTERFACE_LIBS
-           absl::base
-           absl::cord
-           absl::crc32c
-           absl::memory
-           absl::optional
-           absl::span
-           absl::str_format
-           absl::strings
-           absl::time
-           absl::variant)
-    endif()
-  endif()
-endif()
-
 if(ARROW_AZURE)
   if(Azure_SOURCE STREQUAL "SYSTEM")
     list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES})
@@ -992,8 +969,31 @@ if(ARROW_FILESYSTEM)
                                 PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
   endif()
   if(ARROW_GCS)
-    list(APPEND ARROW_FILESYSTEM_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
-    set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
+    set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
+    if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
+      list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
+    elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
+      list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
+      if(absl_SOURCE STREQUAL "SYSTEM")
+        # Bundled google-cloud-cpp with system Abseil needs these for static consumers.
+        list(APPEND
+             ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS
+             absl::base
+             absl::cord
+             absl::crc32c
+             absl::memory
+             absl::optional
+             absl::span
+             absl::str_format
+             absl::strings
+             absl::time
+             absl::variant)
+      endif()
+    endif()
+    set(ARROW_GCS_LIB_SRCS filesystem/gcsfs_module.cc filesystem/gcsfs.cc
+                           filesystem/gcsfs_internal.cc)
+    set_source_files_properties(filesystem/gcsfs_module.cc filesystem/gcsfs.cc
+                                filesystem/gcsfs_internal.cc
                                 PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
     # Suppress documentation warnings from google-cloud-cpp headers
     if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang")
@@ -1009,6 +1009,49 @@ if(ARROW_FILESYSTEM)
                    APPEND
                    PROPERTY COMPILE_OPTIONS "-Wno-error=deprecated-declarations")
     endif()
+    # If libarrow_gcs.a is only built, "pkg-config --cflags --libs
+    # arrow-gcs" outputs build flags for static linking not shared
+    # linking. ARROW_GCS_PC_* except ARROW_GCS_PC_*_PRIVATE are for the
+    # static linking case.
+    if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
+      string(APPEND ARROW_GCS_PC_CFLAGS "${ARROW_GCS_PC_CFLAGS_PRIVATE}")
+      set(ARROW_GCS_PC_CFLAGS_PRIVATE "")
+      set(ARROW_GCS_PC_LIBS "${ARROW_GCS_PC_LIBS_PRIVATE}")
+      set(ARROW_GCS_PC_LIBS_PRIVATE "")
+      set(ARROW_GCS_PC_REQUIRES "${ARROW_GCS_PC_REQUIRES_PRIVATE}")
+      set(ARROW_GCS_PC_REQUIRES_PRIVATE "")
+    else()
+      set(ARROW_GCS_PC_LIBS "")
+      set(ARROW_GCS_PC_REQUIRES "")
+    endif()
+    add_arrow_lib(arrow_gcs
+                  CMAKE_PACKAGE_NAME
+                  ArrowGcs
+                  PKG_CONFIG_NAME
+                  arrow-gcs
+                  SOURCES
+                  ${ARROW_GCS_LIB_SRCS}
+                  SHARED_LINK_LIBS
+                  arrow_shared
+                  SHARED_PRIVATE_LINK_LIBS
+                  google-cloud-cpp::storage
+                  SHARED_INSTALL_INTERFACE_LIBS
+                  Arrow::arrow_shared
+                  STATIC_LINK_LIBS
+                  arrow_static
+                  google-cloud-cpp::storage
+                  STATIC_INSTALL_INTERFACE_LIBS
+                  ${ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS}
+                  SHARED_LINK_FLAGS
+                  ${ARROW_VERSION_SCRIPT_FLAGS}
+                  OUTPUTS
+                  ARROW_GCS_LIBRARIES)
+    foreach(LIB_TARGET ${ARROW_GCS_LIBRARIES})
+      target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_GCS_EXPORTING)
+    endforeach()
+    if(ARROW_BUILD_STATIC AND WIN32)
+      target_compile_definitions(arrow_gcs_static PUBLIC ARROW_GCS_STATIC)
+    endif()
   endif()
   if(ARROW_HDFS)
     list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc filesystem/hdfs_internal.cc)
@@ -1021,11 +1064,6 @@ if(ARROW_FILESYSTEM)
                             PRIVATE ${AZURE_SDK_LINK_LIBRARIES})
     endforeach()
   endif()
-  if(ARROW_GCS)
-    foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
-      target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE google-cloud-cpp::storage)
-    endforeach()
-  endif()
   if(ARROW_HDFS)
     foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
       target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE arrow::hadoop)

cpp/src/arrow/arrow-gcs.pc.in

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
+libdir=@ARROW_PKG_CONFIG_LIBDIR@
+
+Name: Apache Arrow GCS
+Description: Apache Arrow's GCS filesystem implementation.
+Version: @ARROW_VERSION@
+Requires: arrow@ARROW_GCS_PC_REQUIRES@
+Requires.private:@ARROW_GCS_PC_REQUIRES_PRIVATE@
+Libs: -L${libdir} -larrow_gcs@ARROW_GCS_PC_LIBS@
+Libs.private:@ARROW_GCS_PC_LIBS_PRIVATE@
+Cflags:@ARROW_GCS_PC_CFLAGS@
+Cflags.private:@ARROW_GCS_PC_CFLAGS_PRIVATE@

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -43,17 +43,35 @@ if(ARROW_BUILD_BENCHMARKS)
 endif()
 
 if(ARROW_GCS)
+  set(ARROW_GCS_TEST_EXTRA_LINK_LIBS)
+  if(ARROW_TEST_LINKAGE STREQUAL "shared")
+    list(APPEND ARROW_GCS_TEST_EXTRA_LINK_LIBS arrow_gcs_shared)
+  else()
+    list(APPEND ARROW_GCS_TEST_EXTRA_LINK_LIBS arrow_gcs_static)
+  endif()
+  list(APPEND ARROW_GCS_TEST_EXTRA_LINK_LIBS google-cloud-cpp::storage)
   add_arrow_test(gcsfs_test
                  EXTRA_LABELS
                  filesystem
                  EXTRA_LINK_LIBS
-                 google-cloud-cpp::storage)
+                 ${ARROW_GCS_TEST_EXTRA_LINK_LIBS})
   if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang")
     if(TARGET arrow-gcsfs-test)
       target_compile_options(arrow-gcsfs-test PRIVATE -Wno-documentation
                                                       -Wno-documentation-deprecated-sync)
     endif()
   endif()
+  if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED)
+    add_arrow_test(gcsfs_module_test
+                   SOURCES
+                   gcsfs_module_test.cc
+                   EXTRA_LABELS
+                   filesystem
+                   DEFINITIONS
+                   ARROW_GCS_LIBPATH="$<TARGET_FILE:arrow_gcs_shared>"
+                   EXTRA_DEPENDENCIES
+                   arrow_gcs_shared)
+  endif()
 endif()
 
 if(ARROW_AZURE)

cpp/src/arrow/filesystem/filesystem.cc

@@ -28,9 +28,6 @@
 #ifdef ARROW_AZURE
 #  include "arrow/filesystem/azurefs.h"
 #endif
-#ifdef ARROW_GCS
-#  include "arrow/filesystem/gcsfs.h"
-#endif
 #ifdef ARROW_HDFS
 #  include "arrow/filesystem/hdfs.h"
 #endif
@@ -920,16 +917,6 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(
     return Status::NotImplemented(
         "Got Azure Blob File System URI but Arrow compiled without Azure Blob File "
         "System support");
-#endif
-  }
-  if (scheme == "gs" || scheme == "gcs") {
-#ifdef ARROW_GCS
-    ARROW_ASSIGN_OR_RAISE(auto options, GcsOptions::FromUri(uri, out_path));
-    return GcsFileSystem::Make(options, io_context);
-#else
-    return Status::NotImplemented(
-        "Got GCS URI but Arrow compiled "
-        "without GCS support");
 #endif
   }
   if (scheme == "hdfs" || scheme == "viewfs") {