mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-12 08:51:04 +00:00
rename legacy bucket client
This commit is contained in:
1 parent
660cf9f0ac
commit
ce8411f292
49 files changed
+141
-141
No files matched your search
+6
-6
@@ -17,7 +17,7 @@ use tokio::io::AsyncReadExt;
|
||||
use errors::AnyhowResult;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct BucketClient {
|
||||
pub struct LegacyBucketClient {
|
||||
bucket: Bucket,
|
||||
/// If set, put all files under this root path.
|
||||
optional_bucket_root: Option<String>,
|
||||
@@ -25,21 +25,21 @@ pub struct BucketClient {
|
||||
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum BucketClientError {
|
||||
pub enum LegacyBucketClientError {
|
||||
ErrorWithCodeAndMessage { code: u16, message: String },
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BucketClientError {
|
||||
impl std::fmt::Display for LegacyBucketClientError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
match self {
|
||||
BucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message),
|
||||
LegacyBucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Error for BucketClientError {}
|
||||
impl Error for LegacyBucketClientError {}
|
||||
|
||||
|
||||
impl BucketClient {
|
||||
impl LegacyBucketClient {
|
||||
pub fn bucket_name(&self) -> String {
|
||||
self.bucket.name().to_string()
|
||||
}
|
||||
@@ -1,2 +1,2 @@
|
||||
pub mod bucket_client;
|
||||
pub mod legacy_bucket_client;
|
||||
pub mod remote_file_manager;
|
||||
@@ -4,7 +4,7 @@ use tokio::time::Duration;
|
||||
|
||||
use errors::AnyhowResult;
|
||||
|
||||
use crate::bucket_client::BucketClient;
|
||||
use crate::legacy_bucket_client::LegacyBucketClient;
|
||||
|
||||
#[async_trait]
|
||||
#[deprecated(note="this abstraction is too complicated, use another bucket client")]
|
||||
@@ -106,13 +106,13 @@ impl BucketOrchestration {
|
||||
}
|
||||
|
||||
#[deprecated(note="this abstraction is too complicated, use another bucket client")]
|
||||
async fn get_bucket_with_visibility(&self, public: bool) -> AnyhowResult<BucketClient> {
|
||||
async fn get_bucket_with_visibility(&self, public: bool) -> AnyhowResult<LegacyBucketClient> {
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
|
||||
"BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10));
|
||||
let bucket_client = if public {
|
||||
// use public bucket client
|
||||
info!("Configuring public GCS bucket...");
|
||||
BucketClient::create(
|
||||
LegacyBucketClient::create(
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
&self.region_name,
|
||||
@@ -123,7 +123,7 @@ impl BucketOrchestration {
|
||||
)?
|
||||
} else {
|
||||
info!("Configuring private GCS bucket...");
|
||||
BucketClient::create(
|
||||
LegacyBucketClient::create(
|
||||
&self.access_key,
|
||||
&self.secret_key,
|
||||
&self.region_name,
|
||||
|
||||
+6
-6
@@ -20,7 +20,7 @@ pub struct RemoteCloudFileClient {
|
||||
|
||||
impl RemoteCloudFileClient {
|
||||
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
|
||||
pub async fn get_remote_cloud_file_client() -> AnyhowResult<Self> {
|
||||
let bucket_orchestration = match BucketOrchestration::new_bucket_client_from_existing_env() {
|
||||
Ok(client) => client,
|
||||
@@ -34,7 +34,7 @@ impl RemoteCloudFileClient {
|
||||
})
|
||||
}
|
||||
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
|
||||
pub fn new(bucket_orchestration_client: Box<dyn BucketOrchestrationCore>) -> Self {
|
||||
Self {
|
||||
bucket_orchestration_client
|
||||
@@ -42,7 +42,7 @@ impl RemoteCloudFileClient {
|
||||
}
|
||||
|
||||
// Where the to_system_file_path is let sd_checkpoint_path = work_temp_dir.path().join("sd_checkpoint.safetensors"); path and file name and extension
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
|
||||
pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyhowResult<()> {
|
||||
let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details();
|
||||
let file_bucket_directory = FileBucketDirectory::from_existing_bucket_details(remote_cloud_bucket_details);
|
||||
@@ -53,7 +53,7 @@ impl RemoteCloudFileClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
|
||||
pub async fn download_media_file(&self, media_file_path: &MediaFileBucketPath, to_system_file_path: String) -> AnyhowResult<()> {
|
||||
const IS_PUBLIC: bool = true; // NB: Media files are always public
|
||||
let full_remote_cloud_file_path = media_file_path.get_full_object_path_str().to_string();
|
||||
@@ -63,7 +63,7 @@ impl RemoteCloudFileClient {
|
||||
}
|
||||
|
||||
// return error or success with meta data.
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
|
||||
pub async fn upload_file(&self, file_descriptor:Box<dyn FileDescriptor>, from_system_file_path:&str) -> AnyhowResult<FileMetaData> {
|
||||
|
||||
// get file meta data
|
||||
@@ -99,7 +99,7 @@ impl RemoteCloudFileClient {
|
||||
}
|
||||
|
||||
// Retrieve the metadata from the file
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `BucketClient`")]
|
||||
#[deprecated(note="DO NOT USE THIS AWFUL CLOUD BUCKET CLIENT. These abstractions are complicated. See `LegacyBucketClient`")]
|
||||
pub fn get_file_meta_data(system_file_path:&str) -> AnyhowResult<FileMetaData> {
|
||||
let file_size_bytes = file_size(system_file_path.clone())?;
|
||||
let sha256_checksum = sha256_hash_file(system_file_path.clone())?;
|
||||
|
||||
@@ -2,12 +2,12 @@ use std::time::Duration;
|
||||
|
||||
use log::info;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use errors::AnyhowResult;
|
||||
|
||||
pub struct BucketClients {
|
||||
pub public: BucketClient,
|
||||
pub private: BucketClient,
|
||||
pub public: LegacyBucketClient,
|
||||
pub private: LegacyBucketClient,
|
||||
}
|
||||
|
||||
pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
|
||||
@@ -25,7 +25,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
|
||||
|
||||
info!("Configuring public GCS bucket...");
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
@@ -37,7 +37,7 @@ pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
|
||||
|
||||
info!("Configuring private GCS bucket...");
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
let private_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -2,7 +2,7 @@ use log::info;
|
||||
use sqlx::{MySql, Pool};
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
|
||||
use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType;
|
||||
@@ -128,7 +128,7 @@ async fn create_voice_records(
|
||||
Ok((voice_token, dataset_token))
|
||||
}
|
||||
|
||||
async fn seed_file_to_bucket(wav_file: &str, bucket_client: &BucketClient) -> AnyhowResult<MediaFileBucketPath> {
|
||||
async fn seed_file_to_bucket(wav_file: &str, bucket_client: &LegacyBucketClient) -> AnyhowResult<MediaFileBucketPath> {
|
||||
info!("Uploading wav file {} ...", wav_file);
|
||||
|
||||
let public_upload_path = MediaFileBucketPath::generate_new(
|
||||
|
||||
@@ -4,7 +4,7 @@ use log::info;
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
use shared_env_var_config::mysql::env_get_mysql_connection_string_or_default;
|
||||
use enums::by_table::media_files::media_file_type::MediaFileType;
|
||||
@@ -48,7 +48,7 @@ pub async fn main() -> AnyhowResult<()> {
|
||||
|
||||
info!("Configuring GCS bucket...");
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
use sqlx::{MySql, Pool};
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
|
||||
pub struct Deps {
|
||||
pub mysql_development : Pool<MySql>,
|
||||
pub mysql_production : Pool<MySql>,
|
||||
|
||||
pub bucket_development_public: BucketClient,
|
||||
pub bucket_development_private: BucketClient,
|
||||
pub bucket_development_public: LegacyBucketClient,
|
||||
pub bucket_development_private: LegacyBucketClient,
|
||||
|
||||
pub bucket_production_public: BucketClient,
|
||||
pub bucket_production_private: BucketClient,
|
||||
pub bucket_production_public: LegacyBucketClient,
|
||||
pub bucket_production_private: LegacyBucketClient,
|
||||
}
|
||||
@@ -9,7 +9,7 @@ use log::info;
|
||||
use sqlx::{MySql, Pool};
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
|
||||
@@ -68,7 +68,7 @@ async fn get_mysql(env_var_name: &str) -> AnyhowResult<Pool<MySql>> {
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<BucketClient> {
|
||||
pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<LegacyBucketClient> {
|
||||
let env_var_suffix = env_var_suffix.to_uppercase();
|
||||
let access_key = easyenv::get_env_string_required(&format!("ACCESS_KEY_{env_var_suffix}"))?;
|
||||
let secret_key = easyenv::get_env_string_required(&format!("SECRET_KEY_{env_var_suffix}"))?;
|
||||
@@ -84,7 +84,7 @@ pub fn get_bucket_client(env_var_suffix: &str) -> AnyhowResult<BucketClient> {
|
||||
|
||||
info!("Configuring GCS bucket {env_var_suffix} ...");
|
||||
|
||||
Ok(BucketClient::create(
|
||||
Ok(LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -5,7 +5,7 @@ use r2d2_redis::RedisConnectionManager;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
use bootstrap::bootstrap::ContainerEnvironment;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use google_drive_downloader::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg;
|
||||
@@ -33,8 +33,8 @@ pub struct JobState {
|
||||
|
||||
pub redis_pool: r2d2::Pool<RedisConnectionManager>,
|
||||
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub private_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
pub private_bucket_client: LegacyBucketClient,
|
||||
|
||||
pub firehose_publisher: FirehosePublisher,
|
||||
pub badge_granter: BadgeGranter,
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ use anyhow::anyhow;
|
||||
use log::info;
|
||||
use tempdir::TempDir;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use errors::AnyhowResult;
|
||||
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
|
||||
use filesys::file_exists::file_exists;
|
||||
@@ -38,7 +38,7 @@ impl PretrainedHubertModel {
|
||||
|
||||
pub async fn download_if_not_on_filesystem(
|
||||
&self,
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
temp_dir: &TempDir,
|
||||
) -> AnyhowResult<()> {
|
||||
if file_exists(&self.filesystem_path) {
|
||||
|
||||
@@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions;
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use config::common_env::CommonEnv;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
@@ -96,7 +96,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
|
||||
Duration::from_secs(60 * 5));
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
let private_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
@@ -106,7 +106,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
+3
-3
@@ -5,7 +5,7 @@ use log::{debug, error, info, warn};
|
||||
use tempdir::TempDir;
|
||||
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
|
||||
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::rename_across_devices::{rename_across_devices, RenameError};
|
||||
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
|
||||
use tokens::tokens::model_weights::ModelWeightToken;
|
||||
@@ -18,7 +18,7 @@ pub async fn download_package(
|
||||
model_weight_token: ModelWeightToken,
|
||||
weights_file_bucket_directory: &WeightFileBucketDirectory,
|
||||
download_directory: &Path,
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
scoped_tempdir_creator: &ScopedTempDirCreator,
|
||||
force_download: bool,
|
||||
) -> Result<(), ProcessSingleJobError> {
|
||||
@@ -69,7 +69,7 @@ pub async fn do_download_package(
|
||||
model_weight_token: ModelWeightToken,
|
||||
weights_file_bucket_directory: &WeightFileBucketDirectory,
|
||||
download_directory: &Path,
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
temp_dir: &TempDir,
|
||||
force_download: bool,
|
||||
entry: GptSovitsPackageFileType,
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ use zip::ZipArchive;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
|
||||
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::path_to_string::path_to_string;
|
||||
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
|
||||
|
||||
@@ -14,7 +14,7 @@ use crate::job::job_types::gpt_sovits::upload_model::get_gptsovits_zip_entries::
|
||||
|
||||
pub async fn extract_and_upload_gpt_sovits_package_files(
|
||||
zip_container_file_bytes: &[u8],
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
weights_file_bucket_directory: &WeightFileBucketDirectory,
|
||||
) -> Result<GptSovitsPackageDetails, GptSovitsPackageError> {
|
||||
let cursor = Cursor::new(zip_container_file_bytes);
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ use tempdir::TempDir;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
|
||||
use bucket_paths::legacy::typified_paths::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
|
||||
use mysql_queries::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncAnimationAudioSource;
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
@@ -28,7 +28,7 @@ pub struct AudioFile {
|
||||
|
||||
pub struct DownloadAudioFileArgs<'a> {
|
||||
pub audio_source: &'a LipsyncAnimationAudioSource,
|
||||
pub public_bucket_client: &'a BucketClient,
|
||||
pub public_bucket_client: &'a LegacyBucketClient,
|
||||
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
|
||||
pub job: &'a AvailableInferenceJob,
|
||||
pub temp_dir_creator: &'a ScopedTempDirCreator,
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ use tempdir::TempDir;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
|
||||
use mysql_queries::payloads::generic_inference_args::inner_payloads::lipsync_payload::LipsyncAnimationImageSource;
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
@@ -26,7 +26,7 @@ pub struct ImageFile {
|
||||
|
||||
pub struct DownloadImageFileArgs<'a> {
|
||||
pub image_source: &'a LipsyncAnimationImageSource,
|
||||
pub public_bucket_client: &'a BucketClient,
|
||||
pub public_bucket_client: &'a LegacyBucketClient,
|
||||
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
|
||||
pub job: &'a AvailableInferenceJob,
|
||||
pub temp_dir_creator: &'a ScopedTempDirCreator,
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ use tempdir::TempDir;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
|
||||
use mysql_queries::payloads::generic_inference_args::inner_payloads::mocap_payload::MocapVideoSource;
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
@@ -27,7 +27,7 @@ pub struct VideoFile {
|
||||
// TODO: Replace with remote cloud client
|
||||
pub async fn download_video_file(
|
||||
video_source: &MocapVideoSource,
|
||||
public_bucket_client: &BucketClient,
|
||||
public_bucket_client: &LegacyBucketClient,
|
||||
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
|
||||
job: &AvailableInferenceJob,
|
||||
temp_dir_creator: &ScopedTempDirCreator,
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ use log::{error, info, warn};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
|
||||
use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType;
|
||||
@@ -295,7 +295,7 @@ pub struct AudioFile {
|
||||
pub async fn download_audio_from_hash(
|
||||
bucket_hash: &str,
|
||||
name: &str,
|
||||
private_bucket_client: &BucketClient,
|
||||
private_bucket_client: &LegacyBucketClient,
|
||||
path: &PathBuf
|
||||
) -> Result<AudioFile, ProcessSingleJobError> {
|
||||
let unifer = BucketPathUnifier::default_paths();
|
||||
|
||||
+3
-3
@@ -8,7 +8,7 @@ use log::{error, info, warn};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
|
||||
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
|
||||
@@ -244,7 +244,7 @@ pub struct VoiceFile {
|
||||
pub async fn download_voice_embedding_from_hash(
|
||||
bucket_hash: &str,
|
||||
name: &str,
|
||||
private_bucket_client: &BucketClient,
|
||||
private_bucket_client: &LegacyBucketClient,
|
||||
path: &PathBuf
|
||||
) -> Result<VoiceFile, ProcessSingleJobError> {
|
||||
let unifer = BucketPathUnifier::default_paths();
|
||||
@@ -271,7 +271,7 @@ pub async fn download_voice_embedding_from_hash(
|
||||
|
||||
pub async fn download_voice_embedding(
|
||||
voice: &ZsVoice,
|
||||
private_bucket_client: &BucketClient,
|
||||
private_bucket_client: &LegacyBucketClient,
|
||||
download_path: &PathBuf,
|
||||
) -> Result<VoiceFile, ProcessSingleJobError> {
|
||||
|
||||
|
||||
+2
-2
@@ -8,7 +8,7 @@ use log::{error, info, warn};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
|
||||
use enums::by_table::zs_voices::encoding_type::ZsVoiceEncodingType;
|
||||
@@ -316,7 +316,7 @@ pub struct AudioFile {
|
||||
pub async fn download_audio_from_hash(
|
||||
bucket_hash: &str,
|
||||
name: &str,
|
||||
private_bucket_client: &BucketClient,
|
||||
private_bucket_client: &LegacyBucketClient,
|
||||
path: &PathBuf
|
||||
) -> Result<AudioFile, ProcessSingleJobError> {
|
||||
let unifer = BucketPathUnifier::default_paths();
|
||||
|
||||
+3
-3
@@ -8,7 +8,7 @@ use log::{error, info, warn};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
|
||||
use bucket_paths::legacy::typified_paths::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
|
||||
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
|
||||
@@ -260,7 +260,7 @@ pub struct VoiceFile {
|
||||
pub async fn download_voice_embedding_from_hash(
|
||||
bucket_hash: &str,
|
||||
name: &str,
|
||||
private_bucket_client: &BucketClient,
|
||||
private_bucket_client: &LegacyBucketClient,
|
||||
path: &PathBuf
|
||||
) -> Result<VoiceFile, ProcessSingleJobError> {
|
||||
let unifer = BucketPathUnifier::default_paths();
|
||||
@@ -287,7 +287,7 @@ pub async fn download_voice_embedding_from_hash(
|
||||
|
||||
pub async fn download_voice_embedding(
|
||||
voice: &ZsVoice,
|
||||
private_bucket_client: &BucketClient,
|
||||
private_bucket_client: &LegacyBucketClient,
|
||||
download_path: &PathBuf,
|
||||
) -> Result<VoiceFile, ProcessSingleJobError> {
|
||||
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ use std::path::PathBuf;
|
||||
use anyhow::anyhow;
|
||||
use log::{error, info};
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
|
||||
use filesys::file_exists::file_exists;
|
||||
use filesys::rename_across_devices::rename_across_devices;
|
||||
@@ -41,7 +41,7 @@ impl PretrainedHubertModel {
|
||||
|
||||
pub async fn download_if_not_on_filesystem(
|
||||
&self,
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
scoped_tempdir_creator: &ScopedTempDirCreator,
|
||||
) -> Result<(), ProcessSingleJobError> {
|
||||
if file_exists(&self.filesystem_path) {
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ use anyhow::anyhow;
|
||||
use tempdir::TempDir;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
use mysql_queries::queries::model_weights::get::get_weight::RetrievedModelWeight;
|
||||
@@ -19,7 +19,7 @@ pub struct ModelFile {
|
||||
// TODO replace this with remote cloud file manager.
|
||||
pub async fn download_model_file(
|
||||
model_record: &Option<RetrievedModelWeight>,
|
||||
public_bucket_client: &BucketClient,
|
||||
public_bucket_client: &LegacyBucketClient,
|
||||
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
|
||||
job: &AvailableInferenceJob,
|
||||
temp_dir_creator: &ScopedTempDirCreator,
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ use tempdir::TempDir;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
|
||||
use mysql_queries::payloads::generic_inference_args::inner_payloads::videofilter_payload::VideofilterVideoSource;
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
@@ -26,7 +26,7 @@ pub struct VideoFile {
|
||||
// TODO replace this with remote cloud file manager.
|
||||
pub async fn download_video_file(
|
||||
video_source: &VideofilterVideoSource,
|
||||
public_bucket_client: &BucketClient,
|
||||
public_bucket_client: &LegacyBucketClient,
|
||||
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
|
||||
job: &AvailableInferenceJob,
|
||||
temp_dir_creator: &ScopedTempDirCreator,
|
||||
|
||||
+6
-6
@@ -1,5 +1,5 @@
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::file_read_bytes::file_read_bytes;
|
||||
use log::{debug, warn};
|
||||
use std::path::PathBuf;
|
||||
@@ -20,12 +20,12 @@ pub enum BucketUploadMessage {
|
||||
UploadFrame {
|
||||
disk_path: PathBuf,
|
||||
bucket_path: MediaFileBucketPath,
|
||||
bucket_client: BucketClient,
|
||||
bucket_client: LegacyBucketClient,
|
||||
result: oneshot::Sender<BucketUploadResult>
|
||||
},
|
||||
UploadMultipleFrames {
|
||||
requests: Vec<BucketUploadRequest>,
|
||||
bucket_client: BucketClient,
|
||||
bucket_client: LegacyBucketClient,
|
||||
result: oneshot::Sender<Vec<(PathBuf, BucketUploadResult)>>,
|
||||
},
|
||||
}
|
||||
@@ -39,7 +39,7 @@ struct BucketUploadActor {
|
||||
rx: mpsc::Receiver<BucketUploadMessage>,
|
||||
}
|
||||
|
||||
fn handle_single_frame_upload(disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: BucketClient) -> (PathBuf,BucketUploadResult) {
|
||||
fn handle_single_frame_upload(disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: LegacyBucketClient) -> (PathBuf,BucketUploadResult) {
|
||||
let disk_path2 = disk_path.clone();
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
let attempt = handle.block_on(
|
||||
@@ -139,7 +139,7 @@ impl BucketUploadActorHandle {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn upload_frame(&self, disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: BucketClient) -> () {
|
||||
pub async fn upload_frame(&self, disk_path: PathBuf, bucket_path: MediaFileBucketPath, bucket_client: LegacyBucketClient) -> () {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let message = BucketUploadMessage::UploadFrame {
|
||||
disk_path,
|
||||
@@ -151,7 +151,7 @@ impl BucketUploadActorHandle {
|
||||
let _ = rx.await;
|
||||
}
|
||||
|
||||
pub async fn upload_multiple_frames(&self, requests: Vec<BucketUploadRequest>, bucket_client: BucketClient) -> Vec<(PathBuf,BucketUploadResult)> {
|
||||
pub async fn upload_multiple_frames(&self, requests: Vec<BucketUploadRequest>, bucket_client: LegacyBucketClient) -> Vec<(PathBuf,BucketUploadResult)> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let message = BucketUploadMessage::UploadMultipleFrames {
|
||||
requests,
|
||||
|
||||
+8
-8
@@ -14,7 +14,7 @@ use crate::job::job_types::workflow::video_style_transfer::util::preview_process
|
||||
use crate::job::job_types::workflow::video_style_transfer::util::preview_processor::dir_walker::{DirWalker, PathBatch};
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_directory::MediaFileBucketDirectory;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use crockford::crockford_entropy_lower;
|
||||
use redis_schema::keys::inference_job::style_transfer_progress_key::StyleTransferProgressKey;
|
||||
use redis_schema::payloads::inference_job::style_transfer_progress_state::{InferenceProgressDetailsResponse, InferenceStageDetails};
|
||||
@@ -306,7 +306,7 @@ impl StageDirectoryState {
|
||||
}
|
||||
|
||||
|
||||
async fn upload_frame_from_disk(&self, bucket_client: &BucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
|
||||
async fn upload_frame_from_disk(&self, bucket_client: &LegacyBucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
|
||||
let local_file_extension = match disk_path.extension().and_then(|s| s.to_str()) {
|
||||
Some(e) => {
|
||||
if ALLOWED_TYPES_FRAMES.contains(e) {
|
||||
@@ -355,7 +355,7 @@ impl StageDirectoryState {
|
||||
}
|
||||
}
|
||||
|
||||
async fn upload_frame_from_disk_with_retry(&self, bucket_client: &BucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
|
||||
async fn upload_frame_from_disk_with_retry(&self, bucket_client: &LegacyBucketClient, frame_number: u32, disk_path: PathBuf) -> PreviewFrameUpdate {
|
||||
let mut retries = 0;
|
||||
loop {
|
||||
if retries > 3 {
|
||||
@@ -385,7 +385,7 @@ impl StageDirectoryState {
|
||||
}
|
||||
|
||||
|
||||
async fn upload_multiple_frames_from_disk(&self, bucket_client: &BucketClient, frames: Vec<(u32, PathBuf)>) -> Vec<PreviewFrameUpdate> {
|
||||
async fn upload_multiple_frames_from_disk(&self, bucket_client: &LegacyBucketClient, frames: Vec<(u32, PathBuf)>) -> Vec<PreviewFrameUpdate> {
|
||||
let mut bucket_upload_requests = Vec::with_capacity(frames.len());
|
||||
let mut updates = Vec::with_capacity(frames.len());
|
||||
|
||||
@@ -635,7 +635,7 @@ impl PreviewProcessor {
|
||||
let _r: String = redis.set_ex(key.to_string(), serde_json::to_string(&status).unwrap(), expire_at).unwrap();
|
||||
}
|
||||
|
||||
pub async fn start_processing(&mut self, bucket_client: &BucketClient) -> () {
|
||||
pub async fn start_processing(&mut self, bucket_client: &LegacyBucketClient) -> () {
|
||||
let (frames_tx, mut frames_rx) = mpsc::channel(100);
|
||||
let walker = DirWalker::new_with_batch_size(self.base_directory.clone(), frames_tx.clone(), 30);
|
||||
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(5));
|
||||
@@ -693,7 +693,7 @@ impl PreviewProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn process_upload(&mut self, bucket_client: &BucketClient, disk_path: PathBuf) -> () {
|
||||
pub async fn process_upload(&mut self, bucket_client: &LegacyBucketClient, disk_path: PathBuf) -> () {
|
||||
if !self.is_disk_path_valid(&disk_path) {
|
||||
log::warn!("Invalid disk path: {:?}", disk_path);
|
||||
return;
|
||||
@@ -707,7 +707,7 @@ impl PreviewProcessor {
|
||||
};
|
||||
}
|
||||
|
||||
async fn process_batch_upload(&mut self, bucket_client: &BucketClient, disk_paths: PathBatch) -> () {
|
||||
async fn process_batch_upload(&mut self, bucket_client: &LegacyBucketClient, disk_paths: PathBatch) -> () {
|
||||
let mut paths_by_stage: HashMap<PreviewStage, Vec<(u32, PathBuf)>> = HashMap::new();
|
||||
for disk_path in disk_paths.get_paths() {
|
||||
if !self.is_disk_path_valid(&disk_path) {
|
||||
@@ -749,7 +749,7 @@ impl PreviewProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &BucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult {
|
||||
async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &LegacyBucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult {
|
||||
let stage_state = self.stages.get(&stage).unwrap();
|
||||
if stage_state.state == PreviewStageState::UploadComplete {
|
||||
log::debug!("Skipped re-uploading frame: {:?}", disk_path);
|
||||
|
||||
@@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use config::common_env::CommonEnv;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
@@ -117,7 +117,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
|
||||
Duration::from_secs(60 * 5));
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
let private_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
@@ -127,7 +127,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
@@ -137,7 +137,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
let auto_gc_bucket_client = BucketClient::create(
|
||||
let auto_gc_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -10,7 +10,7 @@ use r2d2_redis::RedisConnectionManager;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
use bootstrap::bootstrap::ContainerEnvironment;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
|
||||
@@ -135,9 +135,9 @@ pub struct DatabaseDependencies {
|
||||
}
|
||||
|
||||
pub struct BucketDependencies {
|
||||
pub private_bucket_client: BucketClient,
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub auto_gc_bucket_client: BucketClient,
|
||||
pub private_bucket_client: LegacyBucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
pub auto_gc_bucket_client: LegacyBucketClient,
|
||||
pub bucket_path_unifier: BucketPathUnifier,
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@ use std::path::Path;
|
||||
|
||||
use log::{error, info, warn};
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::file_size::file_size;
|
||||
use filesys::rename_across_devices::{rename_across_devices, RenameError};
|
||||
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
|
||||
@@ -19,7 +19,7 @@ pub struct MaybeDownloadArgs<'a> {
|
||||
pub name_or_description_of_file: &'a str,
|
||||
pub final_filesystem_file_path: &'a Path,
|
||||
pub bucket_object_path: &'a Path,
|
||||
pub bucket_client: &'a BucketClient,
|
||||
pub bucket_client: &'a LegacyBucketClient,
|
||||
pub job_progress_reporter: &'a mut Box<dyn JobProgressReporter>,
|
||||
pub job_progress_update_description: &'a str,
|
||||
pub job_id: i64,
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ use anyhow::anyhow;
|
||||
use async_trait::async_trait;
|
||||
use log::{error, info};
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
|
||||
use filesys::file_exists::file_exists;
|
||||
use filesys::rename_across_devices::{rename_across_devices, RenameError};
|
||||
@@ -29,7 +29,7 @@ pub trait ModelDownloader {
|
||||
|
||||
async fn download_if_not_on_filesystem(
|
||||
&self,
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
scoped_tempdir_creator: &ScopedTempDirCreator,
|
||||
) -> Result<(), ProcessSingleJobError> {
|
||||
let filesystem_path = self.get_filesystem_path();
|
||||
|
||||
+3
-3
@@ -4,7 +4,7 @@ use std::sync::Arc;
|
||||
use log::{error, info, warn};
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use errors::AnyhowResult;
|
||||
use filesys::file_size::file_size;
|
||||
use filesys::rename_across_devices::{rename_across_devices, RenameError};
|
||||
@@ -26,7 +26,7 @@ pub struct ModelWeightsCacheDirectory {
|
||||
pub scoped_tempdir_creator: ScopedTempDirCreator,
|
||||
|
||||
/// A client to download from the public bucket.
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
/// NB: Sometimes the downloader incompletely downloads files. Typically, this is a zero file
|
||||
/// size, but the Rust system may report a non-zero (but small) number of bytes. This should
|
||||
@@ -49,7 +49,7 @@ impl ModelWeightsCacheDirectory {
|
||||
/// Configure and try to create the directory if it doesn't exist.
|
||||
pub fn setup_from_env_and_deps(
|
||||
scoped_temp_dir_creator: &ScopedTempDirCreator,
|
||||
public_bucket_client: &BucketClient,
|
||||
public_bucket_client: &LegacyBucketClient,
|
||||
) -> AnyhowResult<Self> {
|
||||
let directory = easyenv::get_env_pathbuf_or_default(
|
||||
"MODEL_WEIGHTS_CACHE_DIR",
|
||||
|
||||
@@ -28,7 +28,7 @@ use sqlx::mysql::MySqlPoolOptions;
|
||||
use sqlx::MySqlPool;
|
||||
use tempdir::TempDir;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use config::bad_urls::is_bad_tts_model_download_url;
|
||||
use config::common_env::CommonEnv;
|
||||
@@ -74,7 +74,7 @@ struct Downloader {
|
||||
|
||||
pub redis_pool: r2d2::Pool<RedisConnectionManager>,
|
||||
|
||||
pub bucket_client: BucketClient,
|
||||
pub bucket_client: LegacyBucketClient,
|
||||
pub firehose_publisher: FirehosePublisher,
|
||||
pub badge_granter: BadgeGranter,
|
||||
pub google_drive_downloader: GoogleDriveDownloadCommand,
|
||||
@@ -128,7 +128,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
|
||||
Duration::from_secs(60 * 5));
|
||||
|
||||
let bucket_client = BucketClient::create(
|
||||
let bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@ use std::path::Path;
|
||||
|
||||
use log::{info, warn};
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
|
||||
|
||||
use crate::{ProcessSingleJobError, ScopedTempDirCreator};
|
||||
@@ -13,7 +13,7 @@ pub async fn maybe_download_file_from_bucket(
|
||||
name_or_description_of_file: &str,
|
||||
file_path: &Path,
|
||||
bucket_object_path: &Path,
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
job_progress_reporter: &mut Box<dyn JobProgressReporter>,
|
||||
job_progress_update_description: &str,
|
||||
job_id: i64,
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::path::PathBuf;
|
||||
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
@@ -25,8 +25,8 @@ pub struct JobArgs {
|
||||
|
||||
pub job_progress_reporter: Box<dyn JobProgressReporterBuilder>,
|
||||
|
||||
pub private_bucket_client: BucketClient,
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub private_bucket_client: LegacyBucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
pub firehose_publisher: FirehosePublisher,
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ use r2d2_redis::RedisConnectionManager;
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
use sqlx::MySqlPool;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_paths::legacy::old_bespoke_paths::bucket_path_unifier::BucketPathUnifier;
|
||||
use collections::multiple_random_from_vec::multiple_random_from_vec;
|
||||
use config::common_env::CommonEnv;
|
||||
@@ -126,7 +126,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS",
|
||||
Duration::from_secs(60 * 5));
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
let private_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
@@ -136,7 +136,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use gmicloud_client::creds::gmicloud_api_key::GmiCloudApiKey;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
@@ -10,7 +10,7 @@ pub struct JobDependencies {
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
/// Public GCS/S3 bucket for storing generated videos.
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
/// API key for GmiCloud.
|
||||
pub gmicloud_api_key: GmiCloudApiKey,
|
||||
|
||||
@@ -16,7 +16,7 @@ use log::{info, warn};
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
@@ -88,7 +88,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Duration::from_secs(60 * 5),
|
||||
);
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use grok_api_client::creds::grok_api_key::GrokApiKey;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
@@ -14,7 +14,7 @@ pub struct JobDependencies {
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
/// Public GCS/S3 bucket for storing generated videos.
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
/// API key for xAI's Imagine API.
|
||||
pub grok_api_key: GrokApiKey,
|
||||
|
||||
@@ -18,7 +18,7 @@ use sqlx::mysql::MySqlPoolOptions;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use errors::AnyhowResult;
|
||||
use grok_api_client::creds::grok_api_key::GrokApiKey;
|
||||
@@ -89,7 +89,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Duration::from_secs(60 * 5),
|
||||
);
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::sync::Arc;
|
||||
|
||||
use crate::kinovi_version::KinoviVersion;
|
||||
use chrono::Duration;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
use pager::client::pager::Pager;
|
||||
@@ -16,7 +16,7 @@ pub struct JobDependencies {
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
/// Public GCS/S3 bucket for storing generated videos.
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
/// Session credentials for polling seedance2-pro.com.
|
||||
pub seedance2pro_session: Seedance2ProSession,
|
||||
|
||||
@@ -18,7 +18,7 @@ use sqlx::mysql::MySqlPoolOptions;
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
@@ -93,7 +93,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Duration::from_secs(60 * 5),
|
||||
);
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
use pager::client::pager::Pager;
|
||||
@@ -17,7 +17,7 @@ pub struct JobDependencies {
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
/// Public GCS/S3 bucket for downloading source videos and uploading generated thumbnails.
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
pub server_environment: ServerEnvironment,
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ use log::{info, warn};
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
@@ -85,7 +85,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Duration::from_secs(60 * 5),
|
||||
);
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
use server_environment::ServerEnvironment;
|
||||
@@ -9,7 +9,7 @@ pub struct JobDependencies {
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
/// Public GCS/S3 bucket for storing generated splats.
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
|
||||
/// API credentials for World Labs.
|
||||
pub worldlabs_creds: WorldLabsApiCreds,
|
||||
|
||||
@@ -16,7 +16,7 @@ use log::{info, warn};
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
|
||||
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use shared_env_var_config::logging::DEFAULT_RUST_LOG;
|
||||
use errors::AnyhowResult;
|
||||
@@ -85,7 +85,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
Duration::from_secs(60 * 5),
|
||||
);
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ use actix_web::{web, HttpMessage, HttpRequest, HttpResponse};
|
||||
use base64::prelude::BASE64_STANDARD;
|
||||
use base64::Engine;
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use enums::by_table::media_files::media_file_class::MediaFileClass;
|
||||
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
|
||||
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
|
||||
@@ -163,7 +163,7 @@ async fn upload_and_save_image(
|
||||
base64_image: &str,
|
||||
inference_job: &GenericInferenceJobStatus,
|
||||
mysql_pool: &MySqlPool,
|
||||
public_bucket_client: &BucketClient,
|
||||
public_bucket_client: &LegacyBucketClient,
|
||||
) -> AnyhowResult<MediaFileToken> {
|
||||
|
||||
let image_bytes = BASE64_STANDARD.decode(base64_image)?;
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ use once_cell::sync::Lazy;
|
||||
use zip::ZipArchive;
|
||||
|
||||
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use filesys::path_to_string::path_to_string;
|
||||
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
|
||||
use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
|
||||
@@ -40,7 +40,7 @@ pub struct PmxDetails {
|
||||
|
||||
pub async fn extract_and_upload_pmx_files(
|
||||
zip_container_file_bytes: &[u8],
|
||||
bucket_client: &BucketClient,
|
||||
bucket_client: &LegacyBucketClient,
|
||||
prefix: Option<&str>,
|
||||
suffix: Option<&str>
|
||||
) -> Result<PmxDetails, PmxError> {
|
||||
|
||||
@@ -5,7 +5,7 @@ use actix_artcraft::sessions::anonymous_visitor_tracking::avt_cookie_manager::Av
|
||||
use actix_artcraft::sessions::user_sessions::http_user_session_manager::HttpUserSessionManager;
|
||||
use anyhow::anyhow;
|
||||
use chrono::Utc;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use elasticsearch::http::transport::Transport;
|
||||
use elasticsearch::Elasticsearch;
|
||||
use errors::AnyhowResult;
|
||||
@@ -124,17 +124,17 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult<SetupResu
|
||||
let s3_compatible_endpoint_url = easyenv::get_env_string_or_default("S3_COMPATIBLE_ENDPOINT_URL", "https://storage.googleapis.com");
|
||||
let bucket_timeout = easyenv::get_env_duration_seconds_or_default("BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5));
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
let private_bucket_client = LegacyBucketClient::create(
|
||||
&access_key, &secret_key, ®ion_name, &private_bucket_name,
|
||||
&s3_compatible_endpoint_url, None, Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
let public_bucket_client = LegacyBucketClient::create(
|
||||
&access_key, &secret_key, ®ion_name, &public_bucket_name,
|
||||
&s3_compatible_endpoint_url, None, Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
let auto_gc_bucket_client = BucketClient::create(
|
||||
let auto_gc_bucket_client = LegacyBucketClient::create(
|
||||
&access_key, &secret_key, ®ion_name, &gc_enabled_public_bucket_name,
|
||||
&s3_compatible_endpoint_url, None, Some(bucket_timeout),
|
||||
)?;
|
||||
|
||||
@@ -24,7 +24,7 @@ use actix_helpers::middleware::banned_ip_filter::ip_ban_list::ip_ban_list::IpBan
|
||||
use billing_artcraft_component::utils::artcraft_stripe_config::ArtcraftStripeConfigWithClient;
|
||||
use billing_component::stripe::stripe_config::StripeConfig;
|
||||
use chrono::{DateTime, Utc};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use cloud_storage::legacy_bucket_client::LegacyBucketClient;
|
||||
use bucket_client::BucketClient as SeedanceVideoBucketClient;
|
||||
use elasticsearch::Elasticsearch;
|
||||
use memory_caching::arc_ttl_sieve::ArcTtlSieve;
|
||||
@@ -85,9 +85,9 @@ pub struct ServerState {
|
||||
pub firehose_publisher: FirehosePublisher,
|
||||
pub badge_granter: BadgeGranter,
|
||||
|
||||
pub private_bucket_client: BucketClient,
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub auto_gc_bucket_client: BucketClient,
|
||||
pub private_bucket_client: LegacyBucketClient,
|
||||
pub public_bucket_client: LegacyBucketClient,
|
||||
pub auto_gc_bucket_client: LegacyBucketClient,
|
||||
|
||||
/// Optional archive bucket for uploaded Seedance videos. `None` when the
|
||||
/// `SEEDANCE_VIDEO_BUCKET_*` env vars aren't configured.
|
||||
|
||||
Reference in new issue
Block a user