diff --git a/crates/api_clients/cloud_storage/src/bucket_client.rs b/crates/api_clients/cloud_storage/src/legacy_bucket_client.rs similarity index 95% rename from crates/api_clients/cloud_storage/src/bucket_client.rs rename to crates/api_clients/cloud_storage/src/legacy_bucket_client.rs index 40d4a21957..3b5801c47f 100644 --- a/crates/api_clients/cloud_storage/src/bucket_client.rs +++ b/crates/api_clients/cloud_storage/src/legacy_bucket_client.rs @@ -17,7 +17,7 @@ use tokio::io::AsyncReadExt; use errors::AnyhowResult; #[derive(Clone)] -pub struct BucketClient { +pub struct LegacyBucketClient { bucket: Bucket, /// If set, put all files under this root path. optional_bucket_root: Option, @@ -25,21 +25,21 @@ pub struct BucketClient { #[derive(Debug)] -pub enum BucketClientError { +pub enum LegacyBucketClientError { ErrorWithCodeAndMessage { code: u16, message: String }, } -impl std::fmt::Display for BucketClientError { +impl std::fmt::Display for LegacyBucketClientError { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - BucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message), + LegacyBucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message), } } } -impl Error for BucketClientError {} +impl Error for LegacyBucketClientError {} -impl BucketClient { +impl LegacyBucketClient { pub fn bucket_name(&self) -> String { self.bucket.name().to_string() } diff --git a/crates/api_clients/cloud_storage/src/lib.rs b/crates/api_clients/cloud_storage/src/lib.rs index 9b0de5823a..db6553426f 100644 --- a/crates/api_clients/cloud_storage/src/lib.rs +++ b/crates/api_clients/cloud_storage/src/lib.rs @@ -1,2 +1,2 @@ -pub mod bucket_client; +pub mod legacy_bucket_client; pub mod remote_file_manager; \ No newline at end of file diff --git a/crates/api_clients/cloud_storage/src/remote_file_manager/bucket_orchestration.rs b/crates/api_clients/cloud_storage/src/remote_file_manager/bucket_orchestration.rs index 2c728e827b..27bccdbeca 100644 --- a/crates/api_clients/cloud_storage/src/remote_file_manager/bucket_orchestration.rs +++ b/crates/api_clients/cloud_storage/src/remote_file_manager/bucket_orchestration.rs @@ -4,7 +4,7 @@ use tokio::time::Duration; use errors::AnyhowResult; -use crate::bucket_client::BucketClient; +use crate::legacy_bucket_client::LegacyBucketClient; #[async_trait] #[deprecated(note="this abstraction is too complicated, use another bucket client")] @@ -106,13 +106,13 @@ impl BucketOrchestration { } #[deprecated(note="this abstraction is too complicated, use another bucket client")] - async fn get_bucket_with_visibility(&self, public: bool) -> AnyhowResult { + async fn get_bucket_with_visibility(&self, public: bool) -> AnyhowResult { let bucket_timeout = easyenv::get_env_duration_seconds_or_default( "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10)); let bucket_client = if public { // use public bucket client info!("Configuring public GCS bucket..."); - BucketClient::create( + LegacyBucketClient::create( &self.access_key, &self.secret_key, &self.region_name, @@ -123,7 +123,7 @@ impl BucketOrchestration { )? } else { info!("Configuring private GCS bucket..."); - BucketClient::create( + LegacyBucketClient::create( &self.access_key, &self.secret_key, &self.region_name, diff --git a/crates/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs b/crates/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs index 218013164e..771804037a 100644 --- a/crates/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs +++ b/crates/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs @@ -20,7 +20,7 @@ pub struct RemoteCloudFileClient { impl RemoteCloudFileClient { - #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")] + #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")] pub async fn get_remote_cloud_file_client() -> AnyhowResult { let bucket_orchestration = match BucketOrchestration::new_bucket_client_from_existing_env() { Ok(client) => client, @@ -34,7 +34,7 @@ impl RemoteCloudFileClient { }) } - #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")] + #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")] pub fn new(bucket_orchestration_client: Box) -> Self { Self { bucket_orchestration_client @@ -42,7 +42,7 @@ impl RemoteCloudFileClient { } // Where the to_system_file_path is let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors"); path and file name and extension - #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")] + #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")] pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyhowResult<()> { let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details(); let file_bucket_directory = FileBucketDirectory::from_existing_bucket_details(remote_cloud_bucket_details); @@ -53,7 +53,7 @@ impl RemoteCloudFileClient { Ok(()) } - #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")] + #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")] pub async fn download_media_file(&self, media_file_path: &MediaFileBucketPath, to_system_file_path: String) -> AnyhowResult<()> { const IS_PUBLIC: bool = true; // NB: Media files are always public let full_remote_cloud_file_path = media_file_path.get_full_object_path_str().to_string(); @@ -63,7 +63,7 @@ impl RemoteCloudFileClient { } // return error or success with meta data. - #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")] + #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")] pub async fn upload_file(&self, file_descriptor:Box, from_system_file_path:&str) -> AnyhowResult { // get file meta data @@ -99,7 +99,7 @@ impl RemoteCloudFileClient { } // Retrieve the metadata from the file - #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")] + #[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")] pub fn get_file_meta_data(system_file_path:&str) -> AnyhowResult { let file_size_bytes = file_size(system_file_path.clone())?; let sha256_checksum = sha256_hash_file(system_file_path.clone())?; diff --git a/crates/cli/devenv/dev-database-seed/src/bucket_clients.rs b/crates/cli/devenv/dev-database-seed/src/bucket_clients.rs index d0260a95dc..a96f13e713 100644 --- a/crates/cli/devenv/dev-database-seed/src/bucket_clients.rs +++ b/crates/cli/devenv/dev-database-seed/src/bucket_clients.rs @@ -2,12 +2,12 @@ use std::time::Duration; use log::info; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use errors::AnyhowResult; pub struct BucketClients { - pub public: BucketClient, - pub private: BucketClient, + pub public: LegacyBucketClient, + pub private: LegacyBucketClient, } pub fn get_bucket_clients() -> AnyhowResult { @@ -25,7 +25,7 @@ pub fn get_bucket_clients() -> AnyhowResult { info!("Configuring public GCS bucket..."); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, @@ -37,7 +37,7 @@ pub fn get_bucket_clients() -> AnyhowResult { info!("Configuring private GCS bucket..."); - let private_bucket_client = BucketClient::create( + let private_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs b/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs index c8d61de3ca..6e1ab62973 100644 --- a/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs +++ b/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs @@ -2,7 +2,7 @@ use log::info; use sqlx::{MySql, Pool}; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_uploads::media_upload_type::MediaUploadType; use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType; @@ -128,7 +128,7 @@ async fn create_voice_records( Ok((voice_token, dataset_token)) } -async fn seed_file_to_bucket(wav_file: &str, bucket_client: &BucketClient) -> AnyhowResult { +async fn seed_file_to_bucket(wav_file: &str, bucket_client: &LegacyBucketClient) -> AnyhowResult { info!("Uploading wav file {} ...", wav_file); let public_upload_path = MediaFileBucketPath::generate_new( diff --git a/crates/cli/devenv/dev-upload-media-file/src/main.rs b/crates/cli/devenv/dev-upload-media-file/src/main.rs index 41bcd788c7..a2cd1e81a6 100644 --- a/crates/cli/devenv/dev-upload-media-file/src/main.rs +++ b/crates/cli/devenv/dev-upload-media-file/src/main.rs @@ -4,7 +4,7 @@ use log::info; use sqlx::mysql::MySqlPoolOptions; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use shared_env_var_config::logging::DEFAULT_RUST_LOG; use shared_env_var_config::mysql::env_get_mysql_connection_string_or_default; use enums::by_table::media_files::media_file_type::MediaFileType; @@ -48,7 +48,7 @@ pub async fn main() -> AnyhowResult<()> { info!("Configuring GCS bucket..."); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/cli/production/migration-tool/src/deps.rs b/crates/cli/production/migration-tool/src/deps.rs index 0f71d9ab33..4d6cb4047b 100644 --- a/crates/cli/production/migration-tool/src/deps.rs +++ b/crates/cli/production/migration-tool/src/deps.rs @@ -1,14 +1,14 @@ use sqlx::{MySql, Pool}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; pub struct Deps { pub mysql_development : Pool, pub mysql_production : Pool, - pub bucket_development_public: BucketClient, - pub bucket_development_private: BucketClient, + pub bucket_development_public: LegacyBucketClient, + pub bucket_development_private: LegacyBucketClient, - pub bucket_production_public: BucketClient, - pub bucket_production_private: BucketClient, + pub bucket_production_public: LegacyBucketClient, + pub bucket_production_private: LegacyBucketClient, } diff --git a/crates/cli/production/migration-tool/src/main.rs b/crates/cli/production/migration-tool/src/main.rs index 4d3d176eb5..0f37a694e3 100644 --- a/crates/cli/production/migration-tool/src/main.rs +++ b/crates/cli/production/migration-tool/src/main.rs @@ -9,7 +9,7 @@ use log::info; use sqlx::{MySql, Pool}; use sqlx::mysql::MySqlPoolOptions; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use shared_env_var_config::logging::DEFAULT_RUST_LOG; use errors::AnyhowResult; @@ -68,7 +68,7 @@ async fn get_mysql(env_var_name: &str) -> AnyhowResult> { Ok(pool) } -pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult { +pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult { let env_var_suffix = env_var_suffix.to_uppercase(); let access_key = easyenv::get_env_string_required(&format!("ACCESS_KEY_{env_var_suffix}"))?; let secret_key = easyenv::get_env_string_required(&format!("SECRET_KEY_{env_var_suffix}"))?; @@ -84,7 +84,7 @@ pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult { info!("Configuring GCS bucket {env_var_suffix} ..."); - Ok(BucketClient::create( + Ok(LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/deprecated/download_job/src/job_state.rs b/crates/service/job/deprecated/download_job/src/job_state.rs index d535769bda..152b3f475f 100644 --- a/crates/service/job/deprecated/download_job/src/job_state.rs +++ b/crates/service/job/deprecated/download_job/src/job_state.rs @@ -5,7 +5,7 @@ use r2d2_redis::RedisConnectionManager; use sqlx::MySqlPool; use bootstrap::bootstrap::ContainerEnvironment; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use google_drive_downloader::google_drive_download_command::GoogleDriveDownloadCommand; use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg; @@ -33,8 +33,8 @@ pub struct JobState { pub redis_pool: r2d2::Pool, - pub public_bucket_client: BucketClient, - pub private_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, + pub private_bucket_client: LegacyBucketClient, pub firehose_publisher: FirehosePublisher, pub badge_granter: BadgeGranter, diff --git a/crates/service/job/deprecated/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs b/crates/service/job/deprecated/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs index 70a39c1b7c..6029da9eb5 100644 --- a/crates/service/job/deprecated/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs +++ b/crates/service/job/deprecated/download_job/src/job_types/voice_conversion/rvc_v2/pretrained_hubert_model.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use log::info; use tempdir::TempDir; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use errors::AnyhowResult; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; @@ -38,7 +38,7 @@ impl PretrainedHubertModel { pub async fn download_if_not_on_filesystem( &self, - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, temp_dir: &TempDir, ) -> AnyhowResult<()> { if file_exists(&self.filesystem_path) { diff --git a/crates/service/job/deprecated/download_job/src/main.rs b/crates/service/job/deprecated/download_job/src/main.rs index 9362a92fb1..04db0e1bdb 100644 --- a/crates/service/job/deprecated/download_job/src/main.rs +++ b/crates/service/job/deprecated/download_job/src/main.rs @@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions; use tokio::runtime::Runtime; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use config::common_env::CommonEnv; use shared_env_var_config::logging::DEFAULT_RUST_LOG; @@ -96,7 +96,7 @@ async fn main() -> AnyhowResult<()> { let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); - let private_bucket_client = BucketClient::create( + let private_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, @@ -106,7 +106,7 @@ async fn main() -> AnyhowResult<()> { Some(bucket_timeout), )?; - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/model_package/download_package.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/model_package/download_package.rs index 97891b3e20..3db1d6b12e 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/model_package/download_package.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/model_package/download_package.rs @@ -5,7 +5,7 @@ use log::{debug, error, info, warn}; use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory; use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::rename_across_devices::{rename_across_devices, RenameError}; use filesys::file_deletion::safe_delete_directory::safe_delete_directory; use tokens::tokens::model_weights::ModelWeightToken; @@ -18,7 +18,7 @@ pub async fn download_package( model_weight_token: ModelWeightToken, weights_file_bucket_directory: &WeightFileBucketDirectory, download_directory: &Path, - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, scoped_tempdir_creator: &ScopedTempDirCreator, force_download: bool, ) -> Result<(), ProcessSingleJobError> { @@ -69,7 +69,7 @@ pub async fn do_download_package( model_weight_token: ModelWeightToken, weights_file_bucket_directory: &WeightFileBucketDirectory, download_directory: &Path, - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, temp_dir: &TempDir, force_download: bool, entry: GptSovitsPackageFileType, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/upload_model/extract_and_upload_gpt_sovits_package.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/upload_model/extract_and_upload_gpt_sovits_package.rs index 0373be0e6c..a31e9385e1 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/upload_model/extract_and_upload_gpt_sovits_package.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/gpt_sovits/upload_model/extract_and_upload_gpt_sovits_package.rs @@ -5,7 +5,7 @@ use zip::ZipArchive; use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory; use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::path_to_string::path_to_string; use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; @@ -14,7 +14,7 @@ use crate::job::job_types::gpt_sovits::upload_model::get_gptsovits_zip_entries:: pub async fn extract_and_upload_gpt_sovits_package_files( zip_container_file_bytes: &[u8], - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, weights_file_bucket_directory: &WeightFileBucketDirectory, ) -> Result { let cursor = Cursor::new(zip_container_file_bytes); diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs index bea518cf98..ef4040ee50 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_audio_file.rs @@ -8,7 +8,7 @@ use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; use bucket_paths::legacy::typified_paths::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncAnimationAudioSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; @@ -28,7 +28,7 @@ pub struct AudioFile { pub struct DownloadAudioFileArgs<'a> { pub audio_source: &'a LipsyncAnimationAudioSource, - pub public_bucket_client: &'a BucketClient, + pub public_bucket_client: &'a LegacyBucketClient, pub job_progress_reporter: &'a mut Box, pub job: &'a AvailableInferenceJob, pub temp_dir_creator: &'a ScopedTempDirCreator, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs index d5ca04b665..c0b939f96c 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/lipsync/sad_talker/download_image_file.rs @@ -7,7 +7,7 @@ use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncAnimationImageSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; @@ -26,7 +26,7 @@ pub struct ImageFile { pub struct DownloadImageFileArgs<'a> { pub image_source: &'a LipsyncAnimationImageSource, - pub public_bucket_client: &'a BucketClient, + pub public_bucket_client: &'a LegacyBucketClient, pub job_progress_reporter: &'a mut Box, pub job: &'a AvailableInferenceJob, pub temp_dir_creator: &'a ScopedTempDirCreator, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs index d14e19812e..fb8e7184c0 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs @@ -7,7 +7,7 @@ use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::inner_payloads::mocap_payload::MocapVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; @@ -27,7 +27,7 @@ pub struct VideoFile { // TODO: Replace with remote cloud client pub async fn download_video_file( video_source: &MocapVideoSource, - public_bucket_client: &BucketClient, + public_bucket_client: &LegacyBucketClient, job_progress_reporter: &mut Box, job: &AvailableInferenceJob, temp_dir_creator: &ScopedTempDirCreator, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_create_voice.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_create_voice.rs index aec9bdd631..802782aa9a 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_create_voice.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_create_voice.rs @@ -8,7 +8,7 @@ use log::{error, info, warn}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType; @@ -295,7 +295,7 @@ pub struct AudioFile { pub async fn download_audio_from_hash( bucket_hash: &str, name: &str, - private_bucket_client: &BucketClient, + private_bucket_client: &LegacyBucketClient, path: &PathBuf ) -> Result { let unifer = BucketPathUnifier::default_paths(); diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_inference_voice.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_inference_voice.rs index 06bd91995f..a0cba630df 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_inference_voice.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/styletts2/process_job_inference_voice.rs @@ -8,7 +8,7 @@ use log::{error, info, warn}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; @@ -244,7 +244,7 @@ pub struct VoiceFile { pub async fn download_voice_embedding_from_hash( bucket_hash: &str, name: &str, - private_bucket_client: &BucketClient, + private_bucket_client: &LegacyBucketClient, path: &PathBuf ) -> Result { let unifer = BucketPathUnifier::default_paths(); @@ -271,7 +271,7 @@ pub async fn download_voice_embedding_from_hash( pub async fn download_voice_embedding( voice: &ZsVoice, - private_bucket_client: &BucketClient, + private_bucket_client: &LegacyBucketClient, download_path: &PathBuf, ) -> Result { diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs index c9ede5aaa3..777e9eef7f 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_create_voice.rs @@ -8,7 +8,7 @@ use log::{error, info, warn}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType; @@ -316,7 +316,7 @@ pub struct AudioFile { pub async fn download_audio_from_hash( bucket_hash: &str, name: &str, - private_bucket_client: &BucketClient, + private_bucket_client: &LegacyBucketClient, path: &PathBuf ) -> Result { let unifer = BucketPathUnifier::default_paths(); diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs index f4578b55d4..eb3bca1426 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/tts/vall_e_x/process_job_inference_voice.rs @@ -8,7 +8,7 @@ use log::{error, info, warn}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType}; use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; @@ -260,7 +260,7 @@ pub struct VoiceFile { pub async fn download_voice_embedding_from_hash( bucket_hash: &str, name: &str, - private_bucket_client: &BucketClient, + private_bucket_client: &LegacyBucketClient, path: &PathBuf ) -> Result { let unifer = BucketPathUnifier::default_paths(); @@ -287,7 +287,7 @@ pub async fn download_voice_embedding_from_hash( pub async fn download_voice_embedding( voice: &ZsVoice, - private_bucket_client: &BucketClient, + private_bucket_client: &LegacyBucketClient, download_path: &PathBuf, ) -> Result { diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs index 078730b7f8..20bd38865c 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/vc/rvc_v2/pretrained_hubert_model.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use anyhow::anyhow; use log::{error, info}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use filesys::rename_across_devices::rename_across_devices; @@ -41,7 +41,7 @@ impl PretrainedHubertModel { pub async fn download_if_not_on_filesystem( &self, - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, scoped_tempdir_creator: &ScopedTempDirCreator, ) -> Result<(), ProcessSingleJobError> { if file_exists(&self.filesystem_path) { diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs index 0070dbac24..ea1d8c71a8 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::model_weights::get::get_weight::RetrievedModelWeight; @@ -19,7 +19,7 @@ pub struct ModelFile { // TODO replace this with remote cloud file manager. pub async fn download_model_file( model_record: &Option, - public_bucket_client: &BucketClient, + public_bucket_client: &LegacyBucketClient, job_progress_reporter: &mut Box, job: &AvailableInferenceJob, temp_dir_creator: &ScopedTempDirCreator, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs index 6ca478aeec..11cda744c6 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/videofilter/rerender_a_video/download_video_file.rs @@ -7,7 +7,7 @@ use tempdir::TempDir; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use mysql_queries::payloads::generic_inference_args::inner_payloads::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; @@ -26,7 +26,7 @@ pub struct VideoFile { // TODO replace this with remote cloud file manager. pub async fn download_video_file( video_source: &VideofilterVideoSource, - public_bucket_client: &BucketClient, + public_bucket_client: &LegacyBucketClient, job_progress_reporter: &mut Box, job: &AvailableInferenceJob, temp_dir_creator: &ScopedTempDirCreator, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/preview_processor/bucket_uploader.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/preview_processor/bucket_uploader.rs index fa9c1659e0..175378fd41 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/preview_processor/bucket_uploader.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/preview_processor/bucket_uploader.rs @@ -1,5 +1,5 @@ use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::file_read_bytes::file_read_bytes; use log::{debug, warn}; use std::path::PathBuf; @@ -20,12 +20,12 @@ pub enum BucketUploadMessage { UploadFrame { disk_path: PathBuf, bucket_path: MediaFileBucketPath, - bucket_client: BucketClient, + bucket_client: LegacyBucketClient, result: oneshot::Sender }, UploadMultipleFrames { requests: Vec, - bucket_client: BucketClient, + bucket_client: LegacyBucketClient, result: oneshot::Sender>, }, } @@ -39,7 +39,7 @@ struct BucketUploadActor { rx: mpsc::Receiver, } -fn handle_single_frame_upload(disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: BucketClient) -> (PathBuf,BucketUploadResult) { +fn handle_single_frame_upload(disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: LegacyBucketClient) -> (PathBuf,BucketUploadResult) { let disk_path2 = disk_path.clone(); let handle = tokio::runtime::Handle::current(); let attempt = handle.block_on( @@ -139,7 +139,7 @@ impl BucketUploadActorHandle { } } - pub async fn upload_frame(&self, disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: BucketClient) -> () { + pub async fn upload_frame(&self, disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: LegacyBucketClient) -> () { let (tx, rx) = oneshot::channel(); let message = BucketUploadMessage::UploadFrame { disk_path, @@ -151,7 +151,7 @@ impl BucketUploadActorHandle { let _ = rx.await; } - pub async fn upload_multiple_frames(&self, requests: Vec, bucket_client: BucketClient) -> Vec<(PathBuf,BucketUploadResult)> { + pub async fn upload_multiple_frames(&self, requests: Vec, bucket_client: LegacyBucketClient) -> Vec<(PathBuf,BucketUploadResult)> { let (tx, rx) = oneshot::channel(); let message = BucketUploadMessage::UploadMultipleFrames { requests, diff --git a/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/process_preview_updates.rs b/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/process_preview_updates.rs index c74a890802..9f5c254803 100644 --- a/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/process_preview_updates.rs +++ b/crates/service/job/deprecated/inference_job/src/job/job_types/workflow/video_style_transfer/util/process_preview_updates.rs @@ -14,7 +14,7 @@ use crate::job::job_types::workflow::video_style_transfer::util::preview_process use crate::job::job_types::workflow::video_style_transfer::util::preview_processor::dir_walker::{DirWalker, PathBatch}; use bucket_paths::legacy::typified_paths::public::media_files::bucket_directory::MediaFileBucketDirectory; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use crockford::crockford_entropy_lower; use redis_schema::keys::inference_job::style_transfer_progress_key::StyleTransferProgressKey; use redis_schema::payloads::inference_job::style_transfer_progress_state::{InferenceProgressDetailsResponse, InferenceStageDetails}; @@ -306,7 +306,7 @@ impl StageDirectoryState { } - async fn upload_frame_from_disk(&self, bucket_client: &BucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate { + async fn upload_frame_from_disk(&self, bucket_client: &LegacyBucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate { let local_file_extension = match disk_path.extension().and_then(|s| s.to_str()) { Some(e) => { if ALLOWED_TYPES_FRAMES.contains(e) { @@ -355,7 +355,7 @@ impl StageDirectoryState { } } - async fn upload_frame_from_disk_with_retry(&self, bucket_client: &BucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate { + async fn upload_frame_from_disk_with_retry(&self, bucket_client: &LegacyBucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate { let mut retries = 0; loop { if retries > 3 { @@ -385,7 +385,7 @@ impl StageDirectoryState { } - async fn upload_multiple_frames_from_disk(&self, bucket_client: &BucketClient, frames: Vec<(u32, PathBuf)>) -> Vec { + async fn upload_multiple_frames_from_disk(&self, bucket_client: &LegacyBucketClient, frames: Vec<(u32, PathBuf)>) -> Vec { let mut bucket_upload_requests = Vec::with_capacity(frames.len()); let mut updates = Vec::with_capacity(frames.len()); @@ -635,7 +635,7 @@ impl PreviewProcessor { let _r: String = redis.set_ex(key.to_string(), serde_json::to_string(&status).unwrap(), expire_at).unwrap(); } - pub async fn start_processing(&mut self, bucket_client: &BucketClient) -> () { + pub async fn start_processing(&mut self, bucket_client: &LegacyBucketClient) -> () { let (frames_tx, mut frames_rx) = mpsc::channel(100); let walker = DirWalker::new_with_batch_size(self.base_directory.clone(), frames_tx.clone(), 30); let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(5)); @@ -693,7 +693,7 @@ impl PreviewProcessor { } } - pub async fn process_upload(&mut self, bucket_client: &BucketClient, disk_path: PathBuf) -> () { + pub async fn process_upload(&mut self, bucket_client: &LegacyBucketClient, disk_path: PathBuf) -> () { if !self.is_disk_path_valid(&disk_path) { log::warn!("Invalid disk path: {:?}", disk_path); return; @@ -707,7 +707,7 @@ impl PreviewProcessor { }; } - async fn process_batch_upload(&mut self, bucket_client: &BucketClient, disk_paths: PathBatch) -> () { + async fn process_batch_upload(&mut self, bucket_client: &LegacyBucketClient, disk_paths: PathBatch) -> () { let mut paths_by_stage: HashMap> = HashMap::new(); for disk_path in disk_paths.get_paths() { if !self.is_disk_path_valid(&disk_path) { @@ -749,7 +749,7 @@ impl PreviewProcessor { } } - async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &BucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult { + async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &LegacyBucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult { let stage_state = self.stages.get(&stage).unwrap(); if stage_state.state == PreviewStageState::UploadComplete { log::debug!("Skipped re-uploading frame: {:?}", disk_path); diff --git a/crates/service/job/deprecated/inference_job/src/main.rs b/crates/service/job/deprecated/inference_job/src/main.rs index c60534d1e6..68df1bae65 100644 --- a/crates/service/job/deprecated/inference_job/src/main.rs +++ b/crates/service/job/deprecated/inference_job/src/main.rs @@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use config::common_env::CommonEnv; use shared_env_var_config::logging::DEFAULT_RUST_LOG; @@ -117,7 +117,7 @@ async fn main() -> AnyhowResult<()> { let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); - let private_bucket_client = BucketClient::create( + let private_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, @@ -127,7 +127,7 @@ async fn main() -> AnyhowResult<()> { Some(bucket_timeout), )?; - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, @@ -137,7 +137,7 @@ async fn main() -> AnyhowResult<()> { Some(bucket_timeout), )?; - let auto_gc_bucket_client = BucketClient::create( + let auto_gc_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/deprecated/inference_job/src/state/job_dependencies.rs b/crates/service/job/deprecated/inference_job/src/state/job_dependencies.rs index 091288cf09..bace096d3a 100644 --- a/crates/service/job/deprecated/inference_job/src/state/job_dependencies.rs +++ b/crates/service/job/deprecated/inference_job/src/state/job_dependencies.rs @@ -10,7 +10,7 @@ use r2d2_redis::RedisConnectionManager; use sqlx::MySqlPool; use bootstrap::bootstrap::ContainerEnvironment; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; @@ -135,9 +135,9 @@ pub struct DatabaseDependencies { } pub struct BucketDependencies { - pub private_bucket_client: BucketClient, - pub public_bucket_client: BucketClient, - pub auto_gc_bucket_client: BucketClient, + pub private_bucket_client: LegacyBucketClient, + pub public_bucket_client: LegacyBucketClient, + pub auto_gc_bucket_client: LegacyBucketClient, pub bucket_path_unifier: BucketPathUnifier, } diff --git a/crates/service/job/deprecated/inference_job/src/util/downloaders/maybe_download_file_from_bucket.rs b/crates/service/job/deprecated/inference_job/src/util/downloaders/maybe_download_file_from_bucket.rs index 70a9bfd21e..2f36fa9885 100644 --- a/crates/service/job/deprecated/inference_job/src/util/downloaders/maybe_download_file_from_bucket.rs +++ b/crates/service/job/deprecated/inference_job/src/util/downloaders/maybe_download_file_from_bucket.rs @@ -2,7 +2,7 @@ use std::path::Path; use log::{error, info, warn}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::file_size::file_size; use filesys::rename_across_devices::{rename_across_devices, RenameError}; use filesys::file_deletion::safe_delete_directory::safe_delete_directory; @@ -19,7 +19,7 @@ pub struct MaybeDownloadArgs<'a> { pub name_or_description_of_file: &'a str, pub final_filesystem_file_path: &'a Path, pub bucket_object_path: &'a Path, - pub bucket_client: &'a BucketClient, + pub bucket_client: &'a LegacyBucketClient, pub job_progress_reporter: &'a mut Box, pub job_progress_update_description: &'a str, pub job_id: i64, diff --git a/crates/service/job/deprecated/inference_job/src/util/downloaders/model_downloader.rs b/crates/service/job/deprecated/inference_job/src/util/downloaders/model_downloader.rs index 2a791f336f..c44a908bb1 100644 --- a/crates/service/job/deprecated/inference_job/src/util/downloaders/model_downloader.rs +++ b/crates/service/job/deprecated/inference_job/src/util/downloaders/model_downloader.rs @@ -4,7 +4,7 @@ use anyhow::anyhow; use async_trait::async_trait; use log::{error, info}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_exists::file_exists; use filesys::rename_across_devices::{rename_across_devices, RenameError}; @@ -29,7 +29,7 @@ pub trait ModelDownloader { async fn download_if_not_on_filesystem( &self, - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, scoped_tempdir_creator: &ScopedTempDirCreator, ) -> Result<(), ProcessSingleJobError> { let filesystem_path = self.get_filesystem_path(); diff --git a/crates/service/job/deprecated/inference_job/src/util/model_weights_cache/model_weights_cache_directory.rs b/crates/service/job/deprecated/inference_job/src/util/model_weights_cache/model_weights_cache_directory.rs index bb6a09a4a3..923fe05811 100644 --- a/crates/service/job/deprecated/inference_job/src/util/model_weights_cache/model_weights_cache_directory.rs +++ b/crates/service/job/deprecated/inference_job/src/util/model_weights_cache/model_weights_cache_directory.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use log::{error, info, warn}; use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use errors::AnyhowResult; use filesys::file_size::file_size; use filesys::rename_across_devices::{rename_across_devices, RenameError}; @@ -26,7 +26,7 @@ pub struct ModelWeightsCacheDirectory { pub scoped_tempdir_creator: ScopedTempDirCreator, /// A client to download from the public bucket. - pub public_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, /// NB: Sometimes the downloader incompletely downloads files. Typically, this is a zero file /// size, but the Rust system may report a non-zero (but small) number of bytes. This should @@ -49,7 +49,7 @@ impl ModelWeightsCacheDirectory { /// Configure and try to create the directory if it doesn't exist. pub fn setup_from_env_and_deps( scoped_temp_dir_creator: &ScopedTempDirCreator, - public_bucket_client: &BucketClient, + public_bucket_client: &LegacyBucketClient, ) -> AnyhowResult { let directory = easyenv::get_env_pathbuf_or_default( "MODEL_WEIGHTS_CACHE_DIR", diff --git a/crates/service/job/deprecated/tts_download_job/src/main.rs b/crates/service/job/deprecated/tts_download_job/src/main.rs index e8e0b65b76..b6b22fdfcc 100644 --- a/crates/service/job/deprecated/tts_download_job/src/main.rs +++ b/crates/service/job/deprecated/tts_download_job/src/main.rs @@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions; use sqlx::MySqlPool; use tempdir::TempDir; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use config::bad_urls::is_bad_tts_model_download_url; use config::common_env::CommonEnv; @@ -74,7 +74,7 @@ struct Downloader { pub redis_pool: r2d2::Pool, - pub bucket_client: BucketClient, + pub bucket_client: LegacyBucketClient, pub firehose_publisher: FirehosePublisher, pub badge_granter: BadgeGranter, pub google_drive_downloader: GoogleDriveDownloadCommand, @@ -128,7 +128,7 @@ async fn main() -> AnyhowResult<()> { let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); - let bucket_client = BucketClient::create( + let bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/deprecated/tts_inference_job/src/job_steps/download_file_from_bucket.rs b/crates/service/job/deprecated/tts_inference_job/src/job_steps/download_file_from_bucket.rs index 8ff53b2835..27165d0ae0 100644 --- a/crates/service/job/deprecated/tts_inference_job/src/job_steps/download_file_from_bucket.rs +++ b/crates/service/job/deprecated/tts_inference_job/src/job_steps/download_file_from_bucket.rs @@ -2,7 +2,7 @@ use std::path::Path; use log::{info, warn}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; use crate::{ProcessSingleJobError, ScopedTempDirCreator}; @@ -13,7 +13,7 @@ pub async fn maybe_download_file_from_bucket( name_or_description_of_file: &str, file_path: &Path, bucket_object_path: &Path, - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, job_progress_reporter: &mut Box, job_progress_update_description: &str, job_id: i64, diff --git a/crates/service/job/deprecated/tts_inference_job/src/job_steps/job_args.rs b/crates/service/job/deprecated/tts_inference_job/src/job_steps/job_args.rs index acdd711a44..b3453f25ae 100644 --- a/crates/service/job/deprecated/tts_inference_job/src/job_steps/job_args.rs +++ b/crates/service/job/deprecated/tts_inference_job/src/job_steps/job_args.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use sqlx::MySqlPool; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; use jobs_common::job_stats::JobStats; @@ -25,8 +25,8 @@ pub struct JobArgs { pub job_progress_reporter: Box, - pub private_bucket_client: BucketClient, - pub public_bucket_client: BucketClient, + pub private_bucket_client: LegacyBucketClient, + pub public_bucket_client: LegacyBucketClient, pub firehose_publisher: FirehosePublisher, diff --git a/crates/service/job/deprecated/tts_inference_job/src/main.rs b/crates/service/job/deprecated/tts_inference_job/src/main.rs index 94f852092b..b8b9136b1a 100644 --- a/crates/service/job/deprecated/tts_inference_job/src/main.rs +++ b/crates/service/job/deprecated/tts_inference_job/src/main.rs @@ -25,7 +25,7 @@ use r2d2_redis::RedisConnectionManager; use sqlx::mysql::MySqlPoolOptions; use sqlx::MySqlPool; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier; use collections::multiple_random_from_vec::multiple_random_from_vec; use config::common_env::CommonEnv; @@ -126,7 +126,7 @@ async fn main() -> AnyhowResult<()> { let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); - let private_bucket_client = BucketClient::create( + let private_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, @@ -136,7 +136,7 @@ async fn main() -> AnyhowResult<()> { Some(bucket_timeout), )?; - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/gmicloud_job/src/job_dependencies.rs b/crates/service/job/gmicloud_job/src/job_dependencies.rs index 1f3cdfc656..3edf9bbad3 100644 --- a/crates/service/job/gmicloud_job/src/job_dependencies.rs +++ b/crates/service/job/gmicloud_job/src/job_dependencies.rs @@ -1,4 +1,4 @@ -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use gmicloud_client::creds::gmicloud_api_key::GmiCloudApiKey; use jobs_common::job_stats::JobStats; @@ -10,7 +10,7 @@ pub struct JobDependencies { pub mysql_pool: MySqlPool, /// Public GCS/S3 bucket for storing generated videos. - pub public_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, /// API key for GmiCloud. pub gmicloud_api_key: GmiCloudApiKey, diff --git a/crates/service/job/gmicloud_job/src/main.rs b/crates/service/job/gmicloud_job/src/main.rs index 7b2a6b0123..3f010118be 100644 --- a/crates/service/job/gmicloud_job/src/main.rs +++ b/crates/service/job/gmicloud_job/src/main.rs @@ -16,7 +16,7 @@ use log::{info, warn}; use sqlx::mysql::MySqlPoolOptions; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use shared_env_var_config::logging::DEFAULT_RUST_LOG; use errors::AnyhowResult; @@ -88,7 +88,7 @@ async fn main() -> AnyhowResult<()> { Duration::from_secs(60 * 5), ); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/grok_api_job/src/job_dependencies.rs b/crates/service/job/grok_api_job/src/job_dependencies.rs index 70dd6e1526..806fc86c60 100644 --- a/crates/service/job/grok_api_job/src/job_dependencies.rs +++ b/crates/service/job/grok_api_job/src/job_dependencies.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use grok_api_client::creds::grok_api_key::GrokApiKey; use jobs_common::job_stats::JobStats; @@ -14,7 +14,7 @@ pub struct JobDependencies { pub mysql_pool: MySqlPool, /// Public GCS/S3 bucket for storing generated videos. - pub public_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, /// API key for xAI's Imagine API. pub grok_api_key: GrokApiKey, diff --git a/crates/service/job/grok_api_job/src/main.rs b/crates/service/job/grok_api_job/src/main.rs index b161b73850..3a86143946 100644 --- a/crates/service/job/grok_api_job/src/main.rs +++ b/crates/service/job/grok_api_job/src/main.rs @@ -18,7 +18,7 @@ use sqlx::mysql::MySqlPoolOptions; use tokio::sync::Notify; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use errors::AnyhowResult; use grok_api_client::creds::grok_api_key::GrokApiKey; @@ -89,7 +89,7 @@ async fn main() -> AnyhowResult<()> { Duration::from_secs(60 * 5), ); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/seedance2_pro_job/src/job_dependencies.rs b/crates/service/job/seedance2_pro_job/src/job_dependencies.rs index 159a882ada..6960077c4d 100644 --- a/crates/service/job/seedance2_pro_job/src/job_dependencies.rs +++ b/crates/service/job/seedance2_pro_job/src/job_dependencies.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use crate::kinovi_version::KinoviVersion; use chrono::Duration; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use jobs_common::job_stats::JobStats; use pager::client::pager::Pager; @@ -16,7 +16,7 @@ pub struct JobDependencies { pub mysql_pool: MySqlPool, /// Public GCS/S3 bucket for storing generated videos. - pub public_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, /// Session credentials for polling seedance2-pro.com. pub seedance2pro_session: Seedance2ProSession, diff --git a/crates/service/job/seedance2_pro_job/src/main.rs b/crates/service/job/seedance2_pro_job/src/main.rs index d9374a0c8d..bc5fb46e9e 100644 --- a/crates/service/job/seedance2_pro_job/src/main.rs +++ b/crates/service/job/seedance2_pro_job/src/main.rs @@ -18,7 +18,7 @@ use sqlx::mysql::MySqlPoolOptions; use tokio::sync::Notify; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use shared_env_var_config::logging::DEFAULT_RUST_LOG; use errors::AnyhowResult; @@ -93,7 +93,7 @@ async fn main() -> AnyhowResult<()> { Duration::from_secs(60 * 5), ); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/video_thumbnail_job/src/job_dependencies.rs b/crates/service/job/video_thumbnail_job/src/job_dependencies.rs index f7895bf446..760d07c3ea 100644 --- a/crates/service/job/video_thumbnail_job/src/job_dependencies.rs +++ b/crates/service/job/video_thumbnail_job/src/job_dependencies.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use jobs_common::job_stats::JobStats; use pager::client::pager::Pager; @@ -17,7 +17,7 @@ pub struct JobDependencies { pub mysql_pool: MySqlPool, /// Public GCS/S3 bucket for downloading source videos and uploading generated thumbnails. - pub public_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, pub server_environment: ServerEnvironment, diff --git a/crates/service/job/video_thumbnail_job/src/main.rs b/crates/service/job/video_thumbnail_job/src/main.rs index 3ee0d9a9a0..dec94a2b0c 100644 --- a/crates/service/job/video_thumbnail_job/src/main.rs +++ b/crates/service/job/video_thumbnail_job/src/main.rs @@ -17,7 +17,7 @@ use log::{info, warn}; use sqlx::mysql::MySqlPoolOptions; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use shared_env_var_config::logging::DEFAULT_RUST_LOG; use errors::AnyhowResult; @@ -85,7 +85,7 @@ async fn main() -> AnyhowResult<()> { Duration::from_secs(60 * 5), ); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/job/worldlabs_job/src/job_dependencies.rs b/crates/service/job/worldlabs_job/src/job_dependencies.rs index 0978614964..f6deb0d33b 100644 --- a/crates/service/job/worldlabs_job/src/job_dependencies.rs +++ b/crates/service/job/worldlabs_job/src/job_dependencies.rs @@ -1,4 +1,4 @@ -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use jobs_common::job_stats::JobStats; use server_environment::ServerEnvironment; @@ -9,7 +9,7 @@ pub struct JobDependencies { pub mysql_pool: MySqlPool, /// Public GCS/S3 bucket for storing generated splats. - pub public_bucket_client: BucketClient, + pub public_bucket_client: LegacyBucketClient, /// API credentials for World Labs. pub worldlabs_creds: WorldLabsApiCreds, diff --git a/crates/service/job/worldlabs_job/src/main.rs b/crates/service/job/worldlabs_job/src/main.rs index bd52bc7a56..89bfed6590 100644 --- a/crates/service/job/worldlabs_job/src/main.rs +++ b/crates/service/job/worldlabs_job/src/main.rs @@ -16,7 +16,7 @@ use log::{info, warn}; use sqlx::mysql::MySqlPoolOptions; use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use shared_env_var_config::logging::DEFAULT_RUST_LOG; use errors::AnyhowResult; @@ -85,7 +85,7 @@ async fn main() -> AnyhowResult<()> { Duration::from_secs(60 * 5), ); - let public_bucket_client = BucketClient::create( + let public_bucket_client = LegacyBucketClient::create( &access_key, &secret_key, ®ion_name, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs index eda6a50c60..3a008e0d35 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs @@ -11,7 +11,7 @@ use actix_web::{web, HttpMessage, HttpRequest, HttpResponse}; use base64::prelude::BASE64_STANDARD; use base64::Engine; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use enums::by_table::media_files::media_file_class::MediaFileClass; use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; @@ -163,7 +163,7 @@ async fn upload_and_save_image( base64_image: &str, inference_job: &GenericInferenceJobStatus, mysql_pool: &MySqlPool, - public_bucket_client: &BucketClient, + public_bucket_client: &LegacyBucketClient, ) -> AnyhowResult { let image_bytes = BASE64_STANDARD.decode(base64_image)?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_pmx/extract_and_upload_pmx_files.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_pmx/extract_and_upload_pmx_files.rs index 08338f9433..b11d24564f 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_pmx/extract_and_upload_pmx_files.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_pmx/extract_and_upload_pmx_files.rs @@ -7,7 +7,7 @@ use once_cell::sync::Lazy; use zip::ZipArchive; use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use filesys::path_to_string::path_to_string; use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; @@ -40,7 +40,7 @@ pub struct PmxDetails { pub async fn extract_and_upload_pmx_files( zip_container_file_bytes: &[u8], - bucket_client: &BucketClient, + bucket_client: &LegacyBucketClient, prefix: Option<&str>, suffix: Option<&str> ) -> Result { diff --git a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs index 55660c88d0..c02cc1dfdc 100644 --- a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs +++ b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs @@ -5,7 +5,7 @@ use actix_artcraft::sessions::anonymous_visitor_tracking::avt_cookie_manager::Av use actix_artcraft::sessions::user_sessions::http_user_session_manager::HttpUserSessionManager; use anyhow::anyhow; use chrono::Utc; -use cloud_storage::bucket_client::BucketClient; +use cloud_storage::legacy_bucket_client::LegacyBucketClient; use elasticsearch::http::transport::Transport; use elasticsearch::Elasticsearch; use errors::AnyhowResult; @@ -124,17 +124,17 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult