rename legacy bucket client

This commit is contained in:
Brandon Thomas committed 2026-06-19 23:51:46 -04:00
1 parent 660cf9f0ac
commit ce8411f292
49 files changed
+141 -141

No files matched your search

@@ -17,7 +17,7 @@ use tokio::io::AsyncReadExt;
use errors::AnyhowResult;
#[derive(Clone)]
pub struct BucketClient {
pub struct LegacyBucketClient {
bucket: Bucket,
/// If set, put all files under this root path.
optional_bucket_root: Option<String>,
@@ -25,21 +25,21 @@ pub struct BucketClient {
#[derive(Debug)]
pub enum BucketClientError {
pub enum LegacyBucketClientError {
ErrorWithCodeAndMessage { code: u16, message: String },
}
impl std::fmt::Display for BucketClientError {
impl std::fmt::Display for LegacyBucketClientError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
BucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message),
LegacyBucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message),
}
}
}
impl Error for BucketClientError {}
impl Error for LegacyBucketClientError {}
impl BucketClient {
impl LegacyBucketClient {
pub fn bucket_name(&self) -> String {
self.bucket.name().to_string()
}
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod bucket_client;
pub mod legacy_bucket_client;
pub mod remote_file_manager;
@@ -4,7 +4,7 @@ use tokio::time::Duration;
use errors::AnyhowResult;
use crate::bucket_client::BucketClient;
use crate::legacy_bucket_client::LegacyBucketClient;
#[async_trait]
#[deprecated(note="this abstraction is too complicated, use another bucket client")]
@@ -106,13 +106,13 @@ impl BucketOrchestration {
}
#[deprecated(note="this abstraction is too complicated, use another bucket client")]
async fn get_bucket_with_visibility(&self, public: bool) -> AnyhowResult<BucketClient> {
async fn get_bucket_with_visibility(&self, public: bool) -> AnyhowResult<LegacyBucketClient> {
let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
"BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10));
let bucket_client = if public {
// use public bucket client
info!("Configuring public GCS bucket...");
BucketClient::create(
LegacyBucketClient::create(
&self.access_key,
&self.secret_key,
&self.region_name,
@@ -123,7 +123,7 @@ impl BucketOrchestration {
)?
} else {
info!("Configuring private GCS bucket...");
BucketClient::create(
LegacyBucketClient::create(
&self.access_key,
&self.secret_key,
&self.region_name,
@@ -20,7 +20,7 @@ pub struct RemoteCloudFileClient {
impl RemoteCloudFileClient {
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
pub async fn get_remote_cloud_file_client() -> AnyhowResult<Self> {
let bucket_orchestration = match BucketOrchestration::new_bucket_client_from_existing_env() {
Ok(client) => client,
@@ -34,7 +34,7 @@ impl RemoteCloudFileClient {
})
}
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
pub fn new(bucket_orchestration_client: Box<dyn BucketOrchestrationCore>) -> Self {
Self {
bucket_orchestration_client
@@ -42,7 +42,7 @@ impl RemoteCloudFileClient {
}
// Where the to_system_file_path is let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors"); path and file name and extension
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyhowResult<()> {
let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details();
let file_bucket_directory = FileBucketDirectory::from_existing_bucket_details(remote_cloud_bucket_details);
@@ -53,7 +53,7 @@ impl RemoteCloudFileClient {
Ok(())
}
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
pub async fn download_media_file(&self, media_file_path: &MediaFileBucketPath, to_system_file_path: String) -> AnyhowResult<()> {
const IS_PUBLIC: bool = true; // NB: Media files are always public
let full_remote_cloud_file_path = media_file_path.get_full_object_path_str().to_string();
@@ -63,7 +63,7 @@ impl RemoteCloudFileClient {
}
// return error or success with meta data.
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
pub async fn upload_file(&self, file_descriptor:Box<dyn FileDescriptor>, from_system_file_path:&str) -> AnyhowResult<FileMetaData> {
// get file meta data
@@ -99,7 +99,7 @@ impl RemoteCloudFileClient {
}
// Retrieve the metadata from the file
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
pub fn get_file_meta_data(system_file_path:&str) -> AnyhowResult<FileMetaData> {
let file_size_bytes = file_size(system_file_path.clone())?;
let sha256_checksum = sha256_hash_file(system_file_path.clone())?;
@@ -2,12 +2,12 @@ use std::time::Duration;
use log::info;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use errors::AnyhowResult;
pub struct BucketClients {
pub public: BucketClient,
pub private: BucketClient,
pub public: LegacyBucketClient,
pub private: LegacyBucketClient,
}
pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
@@ -25,7 +25,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
info!("Configuring public GCS bucket...");
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -37,7 +37,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
info!("Configuring private GCS bucket...");
let private_bucket_client = BucketClient::create(
let private_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -2,7 +2,7 @@ use log::info;
use sqlx::{MySql, Pool};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType;
@@ -128,7 +128,7 @@ async fn create_voice_records(
Ok((voice_token, dataset_token))
}
async fn seed_file_to_bucket(wav_file: &str, bucket_client: &BucketClient) -> AnyhowResult<MediaFileBucketPath> {
async fn seed_file_to_bucket(wav_file: &str, bucket_client: &LegacyBucketClient) -> AnyhowResult<MediaFileBucketPath> {
info!("Uploading wav file {} ...", wav_file);
let public_upload_path = MediaFileBucketPath::generate_new(
@@ -4,7 +4,7 @@ use log::info;
use sqlx::mysql::MySqlPoolOptions;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
use shared_env_var_config::mysql::env_get_mysql_connection_string_or_default;
use enums::by_table::media_files::media_file_type::MediaFileType;
@@ -48,7 +48,7 @@ pub async fn main() -> AnyhowResult<()> {
info!("Configuring GCS bucket...");
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -1,14 +1,14 @@
use sqlx::{MySql, Pool};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
pub struct Deps {
pub mysql_development : Pool<MySql>,
pub mysql_production : Pool<MySql>,
pub bucket_development_public: BucketClient,
pub bucket_development_private: BucketClient,
pub bucket_development_public: LegacyBucketClient,
pub bucket_development_private: LegacyBucketClient,
pub bucket_production_public: BucketClient,
pub bucket_production_private: BucketClient,
pub bucket_production_public: LegacyBucketClient,
pub bucket_production_private: LegacyBucketClient,
}
@@ -9,7 +9,7 @@ use log::info;
use sqlx::{MySql, Pool};
use sqlx::mysql::MySqlPoolOptions;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
use errors::AnyhowResult;
@@ -68,7 +68,7 @@ async fn get_mysql(env_var_name: &str) -> AnyhowResult<Pool<MySql>> {
Ok(pool)
}
pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<BucketClient> {
pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<LegacyBucketClient> {
let env_var_suffix = env_var_suffix.to_uppercase();
let access_key = easyenv::get_env_string_required(&format!("ACCESS_KEY_{env_var_suffix}"))?;
let secret_key = easyenv::get_env_string_required(&format!("SECRET_KEY_{env_var_suffix}"))?;
@@ -84,7 +84,7 @@ pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<BucketClient> {
info!("Configuring GCS bucket {env_var_suffix} ...");
Ok(BucketClient::create(
Ok(LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -5,7 +5,7 @@ use r2d2_redis::RedisConnectionManager;
use sqlx::MySqlPool;
use bootstrap::bootstrap::ContainerEnvironment;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use google_drive_downloader::google_drive_download_command::GoogleDriveDownloadCommand;
use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg;
@@ -33,8 +33,8 @@ pub struct JobState {
pub redis_pool: r2d2::Pool<RedisConnectionManager>,
pub public_bucket_client: BucketClient,
pub private_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
pub private_bucket_client: LegacyBucketClient,
pub firehose_publisher: FirehosePublisher,
pub badge_granter: BadgeGranter,
@@ -4,7 +4,7 @@ use anyhow::anyhow;
use log::info;
use tempdir::TempDir;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use errors::AnyhowResult;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
use filesys::file_exists::file_exists;
@@ -38,7 +38,7 @@ impl PretrainedHubertModel {
pub async fn download_if_not_on_filesystem(
&self,
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
temp_dir: &TempDir,
) -> AnyhowResult<()> {
if file_exists(&self.filesystem_path) {
@@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions;
use tokio::runtime::Runtime;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use config::common_env::CommonEnv;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
@@ -96,7 +96,7 @@ async fn main() -> AnyhowResult<()> {
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
Duration::from_secs(60 * 5));
let private_bucket_client = BucketClient::create(
let private_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -106,7 +106,7 @@ async fn main() -> AnyhowResult<()> {
Some(bucket_timeout),
)?;
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -5,7 +5,7 @@ use log::{debug, error, info, warn};
use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::rename_across_devices::{rename_across_devices, RenameError};
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
use tokens::tokens::model_weights::ModelWeightToken;
@@ -18,7 +18,7 @@ pub async fn download_package(
model_weight_token: ModelWeightToken,
weights_file_bucket_directory: &WeightFileBucketDirectory,
download_directory: &Path,
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
scoped_tempdir_creator: &ScopedTempDirCreator,
force_download: bool,
) -> Result<(), ProcessSingleJobError> {
@@ -69,7 +69,7 @@ pub async fn do_download_package(
model_weight_token: ModelWeightToken,
weights_file_bucket_directory: &WeightFileBucketDirectory,
download_directory: &Path,
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
temp_dir: &TempDir,
force_download: bool,
entry: GptSovitsPackageFileType,
@@ -5,7 +5,7 @@ use zip::ZipArchive;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
@@ -14,7 +14,7 @@ use crate::job::job_types::gpt_sovits::upload_model::get_gptsovits_zip_entries::
pub async fn extract_and_upload_gpt_sovits_package_files(
zip_container_file_bytes: &[u8],
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
weights_file_bucket_directory: &WeightFileBucketDirectory,
) -> Result<GptSovitsPackageDetails, GptSovitsPackageError> {
let cursor = Cursor::new(zip_container_file_bytes);
@@ -8,7 +8,7 @@ use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use bucket_paths::legacy::typified_paths::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncAnimationAudioSource;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -28,7 +28,7 @@ pub struct AudioFile {
pub struct DownloadAudioFileArgs<'a> {
pub audio_source: &'a LipsyncAnimationAudioSource,
pub public_bucket_client: &'a BucketClient,
pub public_bucket_client: &'a LegacyBucketClient,
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
pub job: &'a AvailableInferenceJob,
pub temp_dir_creator: &'a ScopedTempDirCreator,
@@ -7,7 +7,7 @@ use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncAnimationImageSource;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -26,7 +26,7 @@ pub struct ImageFile {
pub struct DownloadImageFileArgs<'a> {
pub image_source: &'a LipsyncAnimationImageSource,
pub public_bucket_client: &'a BucketClient,
pub public_bucket_client: &'a LegacyBucketClient,
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
pub job: &'a AvailableInferenceJob,
pub temp_dir_creator: &'a ScopedTempDirCreator,
@@ -7,7 +7,7 @@ use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::inner_payloads::mocap_payload::MocapVideoSource;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -27,7 +27,7 @@ pub struct VideoFile {
// TODO: Replace with remote cloud client
pub async fn download_video_file(
video_source: &MocapVideoSource,
public_bucket_client: &BucketClient,
public_bucket_client: &LegacyBucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
temp_dir_creator: &ScopedTempDirCreator,
@@ -8,7 +8,7 @@ use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType;
@@ -295,7 +295,7 @@ pub struct AudioFile {
pub async fn download_audio_from_hash(
bucket_hash: &str,
name: &str,
private_bucket_client: &BucketClient,
private_bucket_client: &LegacyBucketClient,
path: &PathBuf
) -> Result<AudioFile, ProcessSingleJobError> {
let unifer = BucketPathUnifier::default_paths();
@@ -8,7 +8,7 @@ use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
@@ -244,7 +244,7 @@ pub struct VoiceFile {
pub async fn download_voice_embedding_from_hash(
bucket_hash: &str,
name: &str,
private_bucket_client: &BucketClient,
private_bucket_client: &LegacyBucketClient,
path: &PathBuf
) -> Result<VoiceFile, ProcessSingleJobError> {
let unifer = BucketPathUnifier::default_paths();
@@ -271,7 +271,7 @@ pub async fn download_voice_embedding_from_hash(
pub async fn download_voice_embedding(
voice: &ZsVoice,
private_bucket_client: &BucketClient,
private_bucket_client: &LegacyBucketClient,
download_path: &PathBuf,
) -> Result<VoiceFile, ProcessSingleJobError> {
@@ -8,7 +8,7 @@ use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType;
@@ -316,7 +316,7 @@ pub struct AudioFile {
pub async fn download_audio_from_hash(
bucket_hash: &str,
name: &str,
private_bucket_client: &BucketClient,
private_bucket_client: &LegacyBucketClient,
path: &PathBuf
) -> Result<AudioFile, ProcessSingleJobError> {
let unifer = BucketPathUnifier::default_paths();
@@ -8,7 +8,7 @@ use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
@@ -260,7 +260,7 @@ pub struct VoiceFile {
pub async fn download_voice_embedding_from_hash(
bucket_hash: &str,
name: &str,
private_bucket_client: &BucketClient,
private_bucket_client: &LegacyBucketClient,
path: &PathBuf
) -> Result<VoiceFile, ProcessSingleJobError> {
let unifer = BucketPathUnifier::default_paths();
@@ -287,7 +287,7 @@ pub async fn download_voice_embedding_from_hash(
pub async fn download_voice_embedding(
voice: &ZsVoice,
private_bucket_client: &BucketClient,
private_bucket_client: &LegacyBucketClient,
download_path: &PathBuf,
) -> Result<VoiceFile, ProcessSingleJobError> {
@@ -3,7 +3,7 @@ use std::path::PathBuf;
use anyhow::anyhow;
use log::{error, info};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
use filesys::file_exists::file_exists;
use filesys::rename_across_devices::rename_across_devices;
@@ -41,7 +41,7 @@ impl PretrainedHubertModel {
pub async fn download_if_not_on_filesystem(
&self,
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
scoped_tempdir_creator: &ScopedTempDirCreator,
) -> Result<(), ProcessSingleJobError> {
if file_exists(&self.filesystem_path) {
@@ -4,7 +4,7 @@ use anyhow::anyhow;
use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::model_weights::get::get_weight::RetrievedModelWeight;
@@ -19,7 +19,7 @@ pub struct ModelFile {
// TODO replace this with remote cloud file manager.
pub async fn download_model_file(
model_record: &Option<RetrievedModelWeight>,
public_bucket_client: &BucketClient,
public_bucket_client: &LegacyBucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
temp_dir_creator: &ScopedTempDirCreator,
@@ -7,7 +7,7 @@ use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::inner_payloads::videofilter_payload::VideofilterVideoSource;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
@@ -26,7 +26,7 @@ pub struct VideoFile {
// TODO replace this with remote cloud file manager.
pub async fn download_video_file(
video_source: &VideofilterVideoSource,
public_bucket_client: &BucketClient,
public_bucket_client: &LegacyBucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job: &AvailableInferenceJob,
temp_dir_creator: &ScopedTempDirCreator,
@@ -1,5 +1,5 @@
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::file_read_bytes::file_read_bytes;
use log::{debug, warn};
use std::path::PathBuf;
@@ -20,12 +20,12 @@ pub enum BucketUploadMessage {
UploadFrame {
disk_path: PathBuf,
bucket_path: MediaFileBucketPath,
bucket_client: BucketClient,
bucket_client: LegacyBucketClient,
result: oneshot::Sender<BucketUploadResult>
},
UploadMultipleFrames {
requests: Vec<BucketUploadRequest>,
bucket_client: BucketClient,
bucket_client: LegacyBucketClient,
result: oneshot::Sender<Vec<(PathBuf, BucketUploadResult)>>,
},
}
@@ -39,7 +39,7 @@ struct BucketUploadActor {
rx: mpsc::Receiver<BucketUploadMessage>,
}
fn handle_single_frame_upload(disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: BucketClient) -> (PathBuf,BucketUploadResult) {
fn handle_single_frame_upload(disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: LegacyBucketClient) -> (PathBuf,BucketUploadResult) {
let disk_path2 = disk_path.clone();
let handle = tokio::runtime::Handle::current();
let attempt = handle.block_on(
@@ -139,7 +139,7 @@ impl BucketUploadActorHandle {
}
}
pub async fn upload_frame(&self, disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: BucketClient) -> () {
pub async fn upload_frame(&self, disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: LegacyBucketClient) -> () {
let (tx, rx) = oneshot::channel();
let message = BucketUploadMessage::UploadFrame {
disk_path,
@@ -151,7 +151,7 @@ impl BucketUploadActorHandle {
let _ = rx.await;
}
pub async fn upload_multiple_frames(&self, requests: Vec<BucketUploadRequest>, bucket_client: BucketClient) -> Vec<(PathBuf,BucketUploadResult)> {
pub async fn upload_multiple_frames(&self, requests: Vec<BucketUploadRequest>, bucket_client: LegacyBucketClient) -> Vec<(PathBuf,BucketUploadResult)> {
let (tx, rx) = oneshot::channel();
let message = BucketUploadMessage::UploadMultipleFrames {
requests,
@@ -14,7 +14,7 @@ use crate::job::job_types::workflow::video_style_transfer::util::preview_process
use crate::job::job_types::workflow::video_style_transfer::util::preview_processor::dir_walker::{DirWalker, PathBatch};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_directory::MediaFileBucketDirectory;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use crockford::crockford_entropy_lower;
use redis_schema::keys::inference_job::style_transfer_progress_key::StyleTransferProgressKey;
use redis_schema::payloads::inference_job::style_transfer_progress_state::{InferenceProgressDetailsResponse, InferenceStageDetails};
@@ -306,7 +306,7 @@ impl StageDirectoryState {
}
async fn upload_frame_from_disk(&self, bucket_client: &BucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
async fn upload_frame_from_disk(&self, bucket_client: &LegacyBucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
let local_file_extension = match disk_path.extension().and_then(|s| s.to_str()) {
Some(e) => {
if ALLOWED_TYPES_FRAMES.contains(e) {
@@ -355,7 +355,7 @@ impl StageDirectoryState {
}
}
async fn upload_frame_from_disk_with_retry(&self, bucket_client: &BucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
async fn upload_frame_from_disk_with_retry(&self, bucket_client: &LegacyBucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
let mut retries = 0;
loop {
if retries > 3 {
@@ -385,7 +385,7 @@ impl StageDirectoryState {
}
async fn upload_multiple_frames_from_disk(&self, bucket_client: &BucketClient, frames: Vec<(u32, PathBuf)>) -> Vec<PreviewFrameUpdate> {
async fn upload_multiple_frames_from_disk(&self, bucket_client: &LegacyBucketClient, frames: Vec<(u32, PathBuf)>) -> Vec<PreviewFrameUpdate> {
let mut bucket_upload_requests = Vec::with_capacity(frames.len());
let mut updates = Vec::with_capacity(frames.len());
@@ -635,7 +635,7 @@ impl PreviewProcessor {
let _r: String = redis.set_ex(key.to_string(), serde_json::to_string(&status).unwrap(), expire_at).unwrap();
}
pub async fn start_processing(&mut self, bucket_client: &BucketClient) -> () {
pub async fn start_processing(&mut self, bucket_client: &LegacyBucketClient) -> () {
let (frames_tx, mut frames_rx) = mpsc::channel(100);
let walker = DirWalker::new_with_batch_size(self.base_directory.clone(), frames_tx.clone(), 30);
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(5));
@@ -693,7 +693,7 @@ impl PreviewProcessor {
}
}
pub async fn process_upload(&mut self, bucket_client: &BucketClient, disk_path: PathBuf) -> () {
pub async fn process_upload(&mut self, bucket_client: &LegacyBucketClient, disk_path: PathBuf) -> () {
if !self.is_disk_path_valid(&disk_path) {
log::warn!("Invalid disk path: {:?}", disk_path);
return;
@@ -707,7 +707,7 @@ impl PreviewProcessor {
};
}
async fn process_batch_upload(&mut self, bucket_client: &BucketClient, disk_paths: PathBatch) -> () {
async fn process_batch_upload(&mut self, bucket_client: &LegacyBucketClient, disk_paths: PathBatch) -> () {
let mut paths_by_stage: HashMap<PreviewStage, Vec<(u32, PathBuf)>> = HashMap::new();
for disk_path in disk_paths.get_paths() {
if !self.is_disk_path_valid(&disk_path) {
@@ -749,7 +749,7 @@ impl PreviewProcessor {
}
}
async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &BucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult {
async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &LegacyBucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult {
let stage_state = self.stages.get(&stage).unwrap();
if stage_state.state == PreviewStageState::UploadComplete {
log::debug!("Skipped re-uploading frame: {:?}", disk_path);
@@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use config::common_env::CommonEnv;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
@@ -117,7 +117,7 @@ async fn main() -> AnyhowResult<()> {
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
Duration::from_secs(60 * 5));
let private_bucket_client = BucketClient::create(
let private_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -127,7 +127,7 @@ async fn main() -> AnyhowResult<()> {
Some(bucket_timeout),
)?;
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -137,7 +137,7 @@ async fn main() -> AnyhowResult<()> {
Some(bucket_timeout),
)?;
let auto_gc_bucket_client = BucketClient::create(
let auto_gc_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -10,7 +10,7 @@ use r2d2_redis::RedisConnectionManager;
use sqlx::MySqlPool;
use bootstrap::bootstrap::ContainerEnvironment;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
@@ -135,9 +135,9 @@ pub struct DatabaseDependencies {
}
pub struct BucketDependencies {
pub private_bucket_client: BucketClient,
pub public_bucket_client: BucketClient,
pub auto_gc_bucket_client: BucketClient,
pub private_bucket_client: LegacyBucketClient,
pub public_bucket_client: LegacyBucketClient,
pub auto_gc_bucket_client: LegacyBucketClient,
pub bucket_path_unifier: BucketPathUnifier,
}
@@ -2,7 +2,7 @@ use std::path::Path;
use log::{error, info, warn};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::file_size::file_size;
use filesys::rename_across_devices::{rename_across_devices, RenameError};
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
@@ -19,7 +19,7 @@ pub struct MaybeDownloadArgs<'a> {
pub name_or_description_of_file: &'a str,
pub final_filesystem_file_path: &'a Path,
pub bucket_object_path: &'a Path,
pub bucket_client: &'a BucketClient,
pub bucket_client: &'a LegacyBucketClient,
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
pub job_progress_update_description: &'a str,
pub job_id: i64,
@@ -4,7 +4,7 @@ use anyhow::anyhow;
use async_trait::async_trait;
use log::{error, info};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
use filesys::file_exists::file_exists;
use filesys::rename_across_devices::{rename_across_devices, RenameError};
@@ -29,7 +29,7 @@ pub trait ModelDownloader {
async fn download_if_not_on_filesystem(
&self,
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
scoped_tempdir_creator: &ScopedTempDirCreator,
) -> Result<(), ProcessSingleJobError> {
let filesystem_path = self.get_filesystem_path();
@@ -4,7 +4,7 @@ use std::sync::Arc;
use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use errors::AnyhowResult;
use filesys::file_size::file_size;
use filesys::rename_across_devices::{rename_across_devices, RenameError};
@@ -26,7 +26,7 @@ pub struct ModelWeightsCacheDirectory {
pub scoped_tempdir_creator: ScopedTempDirCreator,
/// A client to download from the public bucket.
pub public_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
/// NB: Sometimes the downloader incompletely downloads files. Typically, this is a zero file
/// size, but the Rust system may report a non-zero (but small) number of bytes. This should
@@ -49,7 +49,7 @@ impl ModelWeightsCacheDirectory {
/// Configure and try to create the directory if it doesn't exist.
pub fn setup_from_env_and_deps(
scoped_temp_dir_creator: &ScopedTempDirCreator,
public_bucket_client: &BucketClient,
public_bucket_client: &LegacyBucketClient,
) -> AnyhowResult<Self> {
let directory = easyenv::get_env_pathbuf_or_default(
"MODEL_WEIGHTS_CACHE_DIR",
@@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions;
use sqlx::MySqlPool;
use tempdir::TempDir;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use config::bad_urls::is_bad_tts_model_download_url;
use config::common_env::CommonEnv;
@@ -74,7 +74,7 @@ struct Downloader {
pub redis_pool: r2d2::Pool<RedisConnectionManager>,
pub bucket_client: BucketClient,
pub bucket_client: LegacyBucketClient,
pub firehose_publisher: FirehosePublisher,
pub badge_granter: BadgeGranter,
pub google_drive_downloader: GoogleDriveDownloadCommand,
@@ -128,7 +128,7 @@ async fn main() -> AnyhowResult<()> {
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
Duration::from_secs(60 * 5));
let bucket_client = BucketClient::create(
let bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -2,7 +2,7 @@ use std::path::Path;
use log::{info, warn};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use crate::{ProcessSingleJobError, ScopedTempDirCreator};
@@ -13,7 +13,7 @@ pub async fn maybe_download_file_from_bucket(
name_or_description_of_file: &str,
file_path: &Path,
bucket_object_path: &Path,
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
job_progress_update_description: &str,
job_id: i64,
@@ -2,7 +2,7 @@ use std::path::PathBuf;
use sqlx::MySqlPool;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
use jobs_common::job_stats::JobStats;
@@ -25,8 +25,8 @@ pub struct JobArgs {
pub job_progress_reporter: Box<dyn JobProgressReporterBuilder>,
pub private_bucket_client: BucketClient,
pub public_bucket_client: BucketClient,
pub private_bucket_client: LegacyBucketClient,
pub public_bucket_client: LegacyBucketClient,
pub firehose_publisher: FirehosePublisher,
@@ -25,7 +25,7 @@ use r2d2_redis::RedisConnectionManager;
use sqlx::mysql::MySqlPoolOptions;
use sqlx::MySqlPool;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
use collections::multiple_random_from_vec::multiple_random_from_vec;
use config::common_env::CommonEnv;
@@ -126,7 +126,7 @@ async fn main() -> AnyhowResult<()> {
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
Duration::from_secs(60 * 5));
let private_bucket_client = BucketClient::create(
let private_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -136,7 +136,7 @@ async fn main() -> AnyhowResult<()> {
Some(bucket_timeout),
)?;
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -1,4 +1,4 @@
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use gmicloud_client::creds::gmicloud_api_key::GmiCloudApiKey;
use jobs_common::job_stats::JobStats;
@@ -10,7 +10,7 @@ pub struct JobDependencies {
pub mysql_pool: MySqlPool,
/// Public GCS/S3 bucket for storing generated videos.
pub public_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
/// API key for GmiCloud.
pub gmicloud_api_key: GmiCloudApiKey,
+2 -2
View File
@@ -16,7 +16,7 @@ use log::{info, warn};
use sqlx::mysql::MySqlPoolOptions;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
use errors::AnyhowResult;
@@ -88,7 +88,7 @@ async fn main() -> AnyhowResult<()> {
Duration::from_secs(60 * 5),
);
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -1,6 +1,6 @@
use std::sync::Arc;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use grok_api_client::creds::grok_api_key::GrokApiKey;
use jobs_common::job_stats::JobStats;
@@ -14,7 +14,7 @@ pub struct JobDependencies {
pub mysql_pool: MySqlPool,
/// Public GCS/S3 bucket for storing generated videos.
pub public_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
/// API key for xAI's Imagine API.
pub grok_api_key: GrokApiKey,
+2 -2
View File
@@ -18,7 +18,7 @@ use sqlx::mysql::MySqlPoolOptions;
use tokio::sync::Notify;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use errors::AnyhowResult;
use grok_api_client::creds::grok_api_key::GrokApiKey;
@@ -89,7 +89,7 @@ async fn main() -> AnyhowResult<()> {
Duration::from_secs(60 * 5),
);
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -2,7 +2,7 @@ use std::sync::Arc;
use crate::kinovi_version::KinoviVersion;
use chrono::Duration;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use jobs_common::job_stats::JobStats;
use pager::client::pager::Pager;
@@ -16,7 +16,7 @@ pub struct JobDependencies {
pub mysql_pool: MySqlPool,
/// Public GCS/S3 bucket for storing generated videos.
pub public_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
/// Session credentials for polling seedance2-pro.com.
pub seedance2pro_session: Seedance2ProSession,
@@ -18,7 +18,7 @@ use sqlx::mysql::MySqlPoolOptions;
use tokio::sync::Notify;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
use errors::AnyhowResult;
@@ -93,7 +93,7 @@ async fn main() -> AnyhowResult<()> {
Duration::from_secs(60 * 5),
);
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -1,6 +1,6 @@
use std::path::PathBuf;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use jobs_common::job_stats::JobStats;
use pager::client::pager::Pager;
@@ -17,7 +17,7 @@ pub struct JobDependencies {
pub mysql_pool: MySqlPool,
/// Public GCS/S3 bucket for downloading source videos and uploading generated thumbnails.
pub public_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
pub server_environment: ServerEnvironment,
@@ -17,7 +17,7 @@ use log::{info, warn};
use sqlx::mysql::MySqlPoolOptions;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
use errors::AnyhowResult;
@@ -85,7 +85,7 @@ async fn main() -> AnyhowResult<()> {
Duration::from_secs(60 * 5),
);
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -1,4 +1,4 @@
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use jobs_common::job_stats::JobStats;
use server_environment::ServerEnvironment;
@@ -9,7 +9,7 @@ pub struct JobDependencies {
pub mysql_pool: MySqlPool,
/// Public GCS/S3 bucket for storing generated splats.
pub public_bucket_client: BucketClient,
pub public_bucket_client: LegacyBucketClient,
/// API credentials for World Labs.
pub worldlabs_creds: WorldLabsApiCreds,
+2 -2
View File
@@ -16,7 +16,7 @@ use log::{info, warn};
use sqlx::mysql::MySqlPoolOptions;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
use errors::AnyhowResult;
@@ -85,7 +85,7 @@ async fn main() -> AnyhowResult<()> {
Duration::from_secs(60 * 5),
);
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key,
&secret_key,
&region_name,
@@ -11,7 +11,7 @@ use actix_web::{web, HttpMessage, HttpRequest, HttpResponse};
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
@@ -163,7 +163,7 @@ async fn upload_and_save_image(
base64_image: &str,
inference_job: &GenericInferenceJobStatus,
mysql_pool: &MySqlPool,
public_bucket_client: &BucketClient,
public_bucket_client: &LegacyBucketClient,
) -> AnyhowResult<MediaFileToken> {
let image_bytes = BASE64_STANDARD.decode(base64_image)?;
@@ -7,7 +7,7 @@ use once_cell::sync::Lazy;
use zip::ZipArchive;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
@@ -40,7 +40,7 @@ pub struct PmxDetails {
pub async fn extract_and_upload_pmx_files(
zip_container_file_bytes: &[u8],
bucket_client: &BucketClient,
bucket_client: &LegacyBucketClient,
prefix: Option<&str>,
suffix: Option<&str>
) -> Result<PmxDetails, PmxError> {
@@ -5,7 +5,7 @@ use actix_artcraft::sessions::anonymous_visitor_tracking::avt_cookie_manager::Av
use actix_artcraft::sessions::user_sessions::http_user_session_manager::HttpUserSessionManager;
use anyhow::anyhow;
use chrono::Utc;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use elasticsearch::http::transport::Transport;
use elasticsearch::Elasticsearch;
use errors::AnyhowResult;
@@ -124,17 +124,17 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult<SetupResu
let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL", "https://storage.googleapis.com");
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5));
let private_bucket_client = BucketClient::create(
let private_bucket_client = LegacyBucketClient::create(
&access_key, &secret_key, &region_name, &private_bucket_name,
&s3_compatible_endpoint_url, None, Some(bucket_timeout),
)?;
let public_bucket_client = BucketClient::create(
let public_bucket_client = LegacyBucketClient::create(
&access_key, &secret_key, &region_name, &public_bucket_name,
&s3_compatible_endpoint_url, None, Some(bucket_timeout),
)?;
let auto_gc_bucket_client = BucketClient::create(
let auto_gc_bucket_client = LegacyBucketClient::create(
&access_key, &secret_key, &region_name, &gc_enabled_public_bucket_name,
&s3_compatible_endpoint_url, None, Some(bucket_timeout),
)?;
@@ -24,7 +24,7 @@ use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBan
use billing_artcraft_component::utils::artcraft_stripe_config::ArtcraftStripeConfigWithClient;
use billing_component::stripe::stripe_config::StripeConfig;
use chrono::{DateTime, Utc};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
use bucket_client::BucketClient as SeedanceVideoBucketClient;
use elasticsearch::Elasticsearch;
use memory_caching::arc_ttl_sieve::ArcTtlSieve;
@@ -85,9 +85,9 @@ pub struct ServerState {
pub firehose_publisher: FirehosePublisher,
pub badge_granter: BadgeGranter,
pub private_bucket_client: BucketClient,
pub public_bucket_client: BucketClient,
pub auto_gc_bucket_client: BucketClient,
pub private_bucket_client: LegacyBucketClient,
pub public_bucket_client: LegacyBucketClient,
pub auto_gc_bucket_client: LegacyBucketClient,
/// Optional archive bucket for uploaded Seedance videos. `None` when the
/// `SEEDANCE_VIDEO_BUCKET_*` env vars aren't configured.