Skip to content

MINIFICPP-2703 Update GCP extension to use unified credentials #2086

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lordgamez wants to merge 1 commit into
base: main
Choose a base branch
from
Open

MINIFICPP-2703 Update GCP extension to use unified credentials #2086

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmake/GoogleCloudCpp.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ set(GOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK OFF CACHE INTERNAL macos-openssl
set(BUILD_TESTING OFF CACHE INTERNAL testing-off)
set(GOOGLE_CLOUD_CPP_ENABLE_WERROR OFF CACHE INTERNAL warnings-off)
FetchContent_Declare(google-cloud-cpp
URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.38.0.tar.gz
URL_HASH SHA256=f1493b2dce9b379714342f2be7ccb483d70d13aac09d4a90ae3b4756693b72fc
URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.45.0.tar.gz
URL_HASH SHA256=3d1b5eb696832f9071bf7ef0b3f0c9fd27c1a39d5edcb8a9976c65193319fd01
PATCH_COMMAND "${PC}"
SYSTEM)
if (WIN32)
Expand Down
36 changes: 8 additions & 28 deletions extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,52 +21,32 @@
#include "core/Resource.h"
#include "google/cloud/storage/client.h"
#include "utils/ProcessorConfigUtils.h"

namespace gcs = ::google::cloud::storage;
#include "utils/file/FileUtils.h"

namespace org::apache::nifi::minifi::extensions::gcp {

void GCPCredentialsControllerService::initialize() {
setSupportedProperties(Properties);
}

std::shared_ptr<gcs::oauth2::Credentials> GCPCredentialsControllerService::createDefaultCredentials() const {
auto default_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromDefaultPaths();
if (!default_credentials.ok()) {
logger_->log_error("{}", default_credentials.status().message());
return nullptr;
}
return *default_credentials;
}

std::shared_ptr<gcs::oauth2::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonPath() const {
std::shared_ptr<google::cloud::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonPath() const {
const auto json_path = getProperty(JsonFilePath.name);
if (!json_path) {
logger_->log_error("Missing or invalid {}", JsonFilePath.name);
return nullptr;
}

auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(*json_path);
if (!json_path_credentials.ok()) {
logger_->log_error("{}", json_path_credentials.status().message());
return nullptr;
}
return *json_path_credentials;
return google::cloud::MakeServiceAccountCredentials(utils::file::get_content(*json_path));
Copy link
Contributor

@fgerlits fgerlits Jan 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be useful to check if the file exists, so the user gets a different error when they have mistyped the file name and when the file is empty.

}

std::shared_ptr<gcs::oauth2::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonContents() const {
std::shared_ptr<google::cloud::Credentials> GCPCredentialsControllerService::createCredentialsFromJsonContents() const {
auto json_contents = getProperty(JsonContents.name);
if (!json_contents) {
logger_->log_error("Missing or invalid {}", JsonContents.name);
return nullptr;
}

auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonContents(*json_contents);
if (!json_path_credentials.ok()) {
logger_->log_error("{}", json_path_credentials.status().message());
return nullptr;
}
return *json_path_credentials;
return google::cloud::MakeServiceAccountCredentials(*json_contents);
}

void GCPCredentialsControllerService::onEnable() {
Expand All @@ -79,15 +59,15 @@ void GCPCredentialsControllerService::onEnable() {
credentials_location = CredentialsLocation::USE_DEFAULT_CREDENTIALS;
}
if (*credentials_location == CredentialsLocation::USE_DEFAULT_CREDENTIALS) {
credentials_ = createDefaultCredentials();
credentials_ = google::cloud::MakeGoogleDefaultCredentials();
} else if (*credentials_location == CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS) {
credentials_ = gcs::oauth2::CreateComputeEngineCredentials();
credentials_ = google::cloud::MakeComputeEngineCredentials();
} else if (*credentials_location == CredentialsLocation::USE_JSON_FILE) {
credentials_ = createCredentialsFromJsonPath();
} else if (*credentials_location == CredentialsLocation::USE_JSON_CONTENTS) {
credentials_ = createCredentialsFromJsonContents();
} else if (*credentials_location == CredentialsLocation::USE_ANONYMOUS_CREDENTIALS) {
credentials_ = gcs::oauth2::CreateAnonymousCredentials();
credentials_ = google::cloud::MakeInsecureCredentials();
}
if (!credentials_)
logger_->log_error("Couldn't create valid credentials");
Expand Down
9 changes: 4 additions & 5 deletions extensions/gcp/controllerservices/GCPCredentialsControllerService.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
#include "core/PropertyDefinitionBuilder.h"
#include "utils/Enum.h"

#include "google/cloud/storage/oauth2/credentials.h"
#include "google/cloud/credentials.h"

namespace org::apache::nifi::minifi::extensions::gcp {
enum class CredentialsLocation {
Expand Down Expand Up @@ -113,12 +113,11 @@ class GCPCredentialsControllerService : public core::controller::ControllerServi
[[nodiscard]] const auto& getCredentials() const { return credentials_; }

protected:
[[nodiscard]] std::shared_ptr<google::cloud::storage::oauth2::Credentials> createDefaultCredentials() const;
[[nodiscard]] std::shared_ptr<google::cloud::storage::oauth2::Credentials> createCredentialsFromJsonPath() const;
[[nodiscard]] std::shared_ptr<google::cloud::storage::oauth2::Credentials> createCredentialsFromJsonContents() const;
[[nodiscard]] std::shared_ptr<google::cloud::Credentials> createCredentialsFromJsonPath() const;
[[nodiscard]] std::shared_ptr<google::cloud::Credentials> createCredentialsFromJsonContents() const;


std::shared_ptr<google::cloud::storage::oauth2::Credentials> credentials_;
std::shared_ptr<google::cloud::Credentials> credentials_;
std::shared_ptr<core::logging::Logger> logger_ = core::logging::LoggerFactory<GCPCredentialsControllerService>::getLogger(uuid_);
};
} // namespace org::apache::nifi::minifi::extensions::gcp
14 changes: 9 additions & 5 deletions extensions/gcp/processors/GCSProcessor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ namespace gcs = ::google::cloud::storage;

namespace org::apache::nifi::minifi::extensions::gcp {

std::shared_ptr<google::cloud::storage::oauth2::Credentials> GCSProcessor::getCredentials(core::ProcessContext& context) const {
std::shared_ptr<google::cloud::Credentials> GCSProcessor::getCredentials(core::ProcessContext& context) const {
auto gcp_credentials_controller_service = utils::parseOptionalControllerService<GCPCredentialsControllerService>(context, GCSProcessor::GCPCredentials, getUUID());
if (gcp_credentials_controller_service) {
return gcp_credentials_controller_service->getCredentials();
Expand All @@ -51,10 +51,14 @@ void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessio
}

gcs::Client GCSProcessor::getClient() const {
auto options = gcs::ClientOptions(gcp_credentials_);
if (endpoint_url_)
options.set_endpoint(*endpoint_url_);
return gcs::Client(options, *retry_policy_);
auto options = google::cloud::Options{}
.set<google::cloud::UnifiedCredentialsOption>(gcp_credentials_)
.set<google::cloud::storage::RetryPolicyOption>(retry_policy_);

if (endpoint_url_) {
options.set<gcs::RestEndpointOption>(*endpoint_url_);
}
return gcs::Client(options);
}

} // namespace org::apache::nifi::minifi::extensions::gcp
6 changes: 3 additions & 3 deletions extensions/gcp/processors/GCSProcessor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
#include "minifi-cpp/core/PropertyDefinition.h"
#include "core/PropertyDefinitionBuilder.h"
#include "minifi-cpp/core/PropertyValidator.h"
#include "google/cloud/storage/oauth2/credentials.h"
#include "google/cloud/credentials.h"
#include "google/cloud/storage/client.h"
#include "google/cloud/storage/retry_policy.h"

Expand Down Expand Up @@ -64,10 +64,10 @@ class GCSProcessor : public core::ProcessorImpl {

protected:
virtual google::cloud::storage::Client getClient() const;
std::shared_ptr<google::cloud::storage::oauth2::Credentials> getCredentials(core::ProcessContext& context) const;
std::shared_ptr<google::cloud::Credentials> getCredentials(core::ProcessContext& context) const;

std::optional<std::string> endpoint_url_;
std::shared_ptr<google::cloud::storage::oauth2::Credentials> gcp_credentials_;
std::shared_ptr<google::cloud::Credentials> gcp_credentials_;
google::cloud::storage::RetryPolicyOption::Type retry_policy_ = std::make_shared<google::cloud::storage::LimitedErrorCountRetryPolicy>(6);
};

Expand Down
7 changes: 0 additions & 7 deletions extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,6 @@ class GCPCredentialsTests : public ::testing::Test {
std::shared_ptr<GCPCredentialsControllerService> gcp_credentials_ = std::dynamic_pointer_cast<GCPCredentialsControllerService>(gcp_credentials_node_->getControllerServiceImplementation());
};

TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithoutEnv) {
minifi::utils::Environment::unsetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS");
plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_DEFAULT_CREDENTIALS));
ASSERT_NO_THROW(test_controller_.runSession(plan_));
EXPECT_EQ(nullptr, gcp_credentials_->getCredentials());
}

TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithEnv) {
auto temp_directory = test_controller_.createTempDirectory();
auto path = create_mock_json_file(temp_directory);
Expand Down
Loading