Skip to content

Commit

Permalink
Merge pull request #497 from djmitche/issue489
Browse files Browse the repository at this point in the history 
Use strings for cloud service object names
  • Loading branch information
@djmitche
djmitche authored Dec 8, 2024
2 parents 3b2bb36 + fba0f29 commit ca77420
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 129 deletions.
37 changes: 16 additions & 21 deletions taskchampion/src/server/cloud/aws.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::service::{ObjectInfo, Service};
use super::service::{validate_object_name, ObjectInfo, Service};
use crate::errors::Result;
use aws_config::{
meta::region::RegionProviderChain, profile::ProfileFileCredentialsProvider, BehaviorVersion,
Expand Down Expand Up @@ -107,11 +107,6 @@ impl AwsService {
}
}

/// Convert an object name from bytes to a string.
fn name_to_string(name: &[u8]) -> String {
String::from_utf8(name.to_vec()).expect("non-UTF8 object name")
}

/// Convert an error that can be converted to `s3::Error` (but not [`crate::Error`]) into
/// `s3::Error`. One such error is SdkError, which has type parameters that are difficult to
/// constrain in order to write `From<SdkError<..>> for crate::Error`.
Expand Down Expand Up @@ -140,9 +135,9 @@ async fn get_body(get_res: GetObjectOutput) -> Result<Vec<u8>> {
}

impl Service for AwsService {
fn put(&mut self, name: &[u8], value: &[u8]) -> Result<()> {
fn put(&mut self, name: &str, value: &[u8]) -> Result<()> {
self.block_on(async {
let name = name_to_string(name);
validate_object_name(name);
self.client
.put_object()
.bucket(self.bucket.clone())
Expand All @@ -155,9 +150,9 @@ impl Service for AwsService {
})
}

fn get(&mut self, name: &[u8]) -> Result<Option<Vec<u8>>> {
fn get(&mut self, name: &str) -> Result<Option<Vec<u8>>> {
self.block_on(async {
let name = name_to_string(name);
validate_object_name(name);
let Some(get_res) = if_key_exists(
self.client
.get_object()
Expand All @@ -174,9 +169,9 @@ impl Service for AwsService {
})
}

fn del(&mut self, name: &[u8]) -> Result<()> {
fn del(&mut self, name: &str) -> Result<()> {
self.block_on(async {
let name = name_to_string(name);
validate_object_name(name);
self.client
.delete_object()
.bucket(self.bucket.clone())
Expand All @@ -188,29 +183,29 @@ impl Service for AwsService {
})
}

fn list<'a>(&'a mut self, prefix: &[u8]) -> Box<dyn Iterator<Item = Result<ObjectInfo>> + 'a> {
let prefix = name_to_string(prefix);
fn list<'a>(&'a mut self, prefix: &str) -> Box<dyn Iterator<Item = Result<ObjectInfo>> + 'a> {
validate_object_name(prefix);
Box::new(ObjectIterator {
service: self,
prefix,
prefix: prefix.to_string(),
last_response: None,
next_index: 0,
})
}

fn compare_and_swap(
&mut self,
name: &[u8],
name: &str,
existing_value: Option<Vec<u8>>,
new_value: Vec<u8>,
) -> Result<bool> {
self.block_on(async {
let name = name_to_string(name);
validate_object_name(name);
let get_res = if_key_exists(
self.client
.get_object()
.bucket(self.bucket.clone())
.key(name.clone())
.key(name)
.send()
.await
.map_err(aws_err),
Expand Down Expand Up @@ -244,7 +239,7 @@ impl Service for AwsService {
self.client
.delete_object()
.bucket(self.bucket.clone())
.key(name.clone())
.key(name)
.send()
.await
.map_err(aws_err)?;
Expand All @@ -258,7 +253,7 @@ impl Service for AwsService {
self.client
.put_object()
.bucket(self.bucket.clone())
.key(name.clone())
.key(name)
.body(b"CHANGED".to_vec().into())
.send()
.await
Expand Down Expand Up @@ -356,7 +351,7 @@ impl Iterator for ObjectIterator<'_> {
let creation: u64 = creation.try_into().unwrap_or(0);
let name = obj.key.as_ref().expect("object has no key").clone();
return Some(Ok(ObjectInfo {
name: name.as_bytes().to_vec(),
name: name.clone(),
creation,
}));
} else if result.next_continuation_token.is_some() {
Expand Down
44 changes: 23 additions & 21 deletions taskchampion/src/server/cloud/gcp.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::service::{ObjectInfo, Service};
use super::service::{validate_object_name, ObjectInfo, Service};
use crate::errors::Result;
use google_cloud_storage::client::google_cloud_auth::credentials::CredentialsFile;
use google_cloud_storage::client::{Client, ClientConfig};
Expand Down Expand Up @@ -45,9 +45,10 @@ impl GcpService {
}

impl Service for GcpService {
fn put(&mut self, name: &[u8], value: &[u8]) -> Result<()> {
let name = String::from_utf8(name.to_vec()).expect("non-UTF8 object name");
let upload_type = objects::upload::UploadType::Simple(objects::upload::Media::new(name));
fn put(&mut self, name: &str, value: &[u8]) -> Result<()> {
validate_object_name(name);
let upload_type =
objects::upload::UploadType::Simple(objects::upload::Media::new(name.to_string()));
self.rt.block_on(self.client.upload_object(
&objects::upload::UploadObjectRequest {
bucket: self.bucket.clone(),
Expand All @@ -59,12 +60,12 @@ impl Service for GcpService {
Ok(())
}

fn get(&mut self, name: &[u8]) -> Result<Option<Vec<u8>>> {
let name = String::from_utf8(name.to_vec()).expect("non-UTF8 object name");
fn get(&mut self, name: &str) -> Result<Option<Vec<u8>>> {
validate_object_name(name);
let download_res = self.rt.block_on(self.client.download_object(
&objects::get::GetObjectRequest {
bucket: self.bucket.clone(),
object: name,
object: name.to_string(),
..Default::default()
},
&objects::download::Range::default(),
Expand All @@ -76,12 +77,12 @@ impl Service for GcpService {
}
}

fn del(&mut self, name: &[u8]) -> Result<()> {
let name = String::from_utf8(name.to_vec()).expect("non-UTF8 object name");
fn del(&mut self, name: &str) -> Result<()> {
validate_object_name(name);
let del_res = self.rt.block_on(self.client.delete_object(
&objects::delete::DeleteObjectRequest {
bucket: self.bucket.clone(),
object: name,
object: name.to_string(),
..Default::default()
},
));
Expand All @@ -91,28 +92,28 @@ impl Service for GcpService {
Ok(())
}

fn list<'a>(&'a mut self, prefix: &[u8]) -> Box<dyn Iterator<Item = Result<ObjectInfo>> + 'a> {
let prefix = String::from_utf8(prefix.to_vec()).expect("non-UTF8 object prefix");
fn list<'a>(&'a mut self, prefix: &str) -> Box<dyn Iterator<Item = Result<ObjectInfo>> + 'a> {
validate_object_name(prefix);
Box::new(ObjectIterator {
service: self,
prefix,
prefix: prefix.to_string(),
last_response: None,
next_index: 0,
})
}

fn compare_and_swap(
&mut self,
name: &[u8],
name: &str,
existing_value: Option<Vec<u8>>,
new_value: Vec<u8>,
) -> Result<bool> {
let name = String::from_utf8(name.to_vec()).expect("non-UTF8 object name");
validate_object_name(name);
let get_res = self
.rt
.block_on(self.client.get_object(&objects::get::GetObjectRequest {
bucket: self.bucket.clone(),
object: name.clone(),
object: name.to_string(),
..Default::default()
}));
// Determine the object's generation. See https://cloud.google.com/storage/docs/metadata#generation-number
Expand All @@ -132,7 +133,7 @@ impl Service for GcpService {
let data = self.rt.block_on(self.client.download_object(
&objects::get::GetObjectRequest {
bucket: self.bucket.clone(),
object: name.clone(),
object: name.to_string(),
// Fetch the same generation.
generation: Some(generation),
..Default::default()
Expand All @@ -152,7 +153,7 @@ impl Service for GcpService {
let del_res = self.rt.block_on(self.client.delete_object(
&objects::delete::DeleteObjectRequest {
bucket: self.bucket.clone(),
object: name.clone(),
object: name.to_string(),
..Default::default()
},
));
Expand All @@ -167,7 +168,7 @@ impl Service for GcpService {
if name.ends_with("-racing-put") {
println!("changing object {name}");
let upload_type =
objects::upload::UploadType::Simple(objects::upload::Media::new(name.clone()));
objects::upload::UploadType::Simple(objects::upload::Media::new(name.to_string()));
self.rt.block_on(self.client.upload_object(
&objects::upload::UploadObjectRequest {
bucket: self.bucket.clone(),
Expand All @@ -179,7 +180,8 @@ impl Service for GcpService {
}

// Finally, put the new value with a condition that the generation hasn't changed.
let upload_type = objects::upload::UploadType::Simple(objects::upload::Media::new(name));
let upload_type =
objects::upload::UploadType::Simple(objects::upload::Media::new(name.to_string()));
let upload_res = self.rt.block_on(self.client.upload_object(
&objects::upload::UploadObjectRequest {
bucket: self.bucket.clone(),
Expand Down Expand Up @@ -251,7 +253,7 @@ impl Iterator for ObjectIterator<'_> {
let creation = obj.time_created.map(|t| t.unix_timestamp()).unwrap_or(0);
let creation: u64 = creation.try_into().unwrap_or(0);
return Some(Ok(ObjectInfo {
name: obj.name.as_bytes().to_vec(),
name: obj.name.clone(),
creation,
}));
} else if result.next_page_token.is_some() {
Expand Down
Loading

0 comments on commit ca77420

Please sign in to comment.