added new s3 params

CHANGELOG.md

@@ -27,9 +27,13 @@
 
 Released on ??
 
+- **Added support for S3 compatible backends**
+  - Changed `AWS S3` to `S3` in ui
+  - Added new `endpoint` and `new-path-style` to s3 connection parameters
 - Bugfix:
   - [Issue 92](https://github.com/veeso/termscp/issues/92): updated ssh2-config to 0.1.3, which solves this issue.
 - Dependencies:
+  - Updated `remotefs-rs-aws-s3` to `0.2.0`
   - Updated `tui-realm-stdlib` to `1.1.6`
 
 ## 0.8.0

Cargo.lock

@@ -1794,9 +1794,9 @@ dependencies = [
 
 [[package]]
 name = "remotefs-aws-s3"
-version = "0.1.0"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36b130b79d6ade3821554c46a87fd54dea024255daa5ecac5e9f68e2a4d3230a"
+checksum = "9939849e0b3895b07f258458d0fa5cd4632c78ffc517e66e1f758ddf23990542"
 dependencies = [
  "chrono",
  "log",

Cargo.toml

@@ -48,7 +48,7 @@ open = "2.0.2"
 rand = "0.8.4"
 regex = "1.5.4"
 remotefs = "^0.2.0"
-remotefs-aws-s3 = "^0.1.0"
+remotefs-aws-s3 = "^0.2.0"
 remotefs-ftp = { version = "^0.1.0", features = [ "secure" ] }
 remotefs-ssh = "^0.1.0"
 rpassword = "5.0.1"

src/config/bookmarks.rs

@@ -64,10 +64,13 @@ pub struct Bookmark {
 #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Default)]
 pub struct S3Params {
     pub bucket: String,
-    pub region: String,
+    pub region: Option<String>,
+    pub endpoint: Option<String>,
     pub profile: Option<String>,
     pub access_key: Option<String>,
     pub secret_access_key: Option<String>,
+    /// NOTE: there are no session token and security token since they are always temporary
+    pub new_path_style: Option<bool>,
 }
 
 // -- impls
@@ -123,9 +126,11 @@ impl From<AwsS3Params> for S3Params {
         S3Params {
             bucket: params.bucket_name,
             region: params.region,
+            endpoint: params.endpoint,
             profile: params.profile,
             access_key: params.access_key,
             secret_access_key: params.secret_access_key,
+            new_path_style: Some(params.new_path_style),
         }
     }
 }
@@ -133,8 +138,10 @@ impl From<AwsS3Params> for S3Params {
 impl From<S3Params> for AwsS3Params {
     fn from(params: S3Params) -> Self {
         AwsS3Params::new(params.bucket, params.region, params.profile)
+            .endpoint(params.endpoint)
             .access_key(params.access_key)
             .secret_access_key(params.secret_access_key)
+            .new_path_style(params.new_path_style.unwrap_or(false))
     }
 }
 
@@ -234,7 +241,7 @@ mod tests {
     #[test]
     fn bookmark_from_s3_ftparams() {
         let params = ProtocolParams::AwsS3(
-            AwsS3Params::new("omar", "eu-west-1", Some("test"))
+            AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))
                 .access_key(Some("pippo"))
                 .secret_access_key(Some("pluto")),
         );
@@ -248,7 +255,7 @@ mod tests {
         assert!(bookmark.password.is_none());
         let s3: &S3Params = bookmark.s3.as_ref().unwrap();
         assert_eq!(s3.bucket.as_str(), "omar");
-        assert_eq!(s3.region.as_str(), "eu-west-1");
+        assert_eq!(s3.region.as_deref().unwrap(), "eu-west-1");
         assert_eq!(s3.profile.as_deref().unwrap(), "test");
         assert_eq!(s3.access_key.as_deref().unwrap(), "pippo");
         assert_eq!(s3.secret_access_key.as_deref().unwrap(), "pluto");
@@ -283,19 +290,23 @@ mod tests {
             password: None,
             s3: Some(S3Params {
                 bucket: String::from("veeso"),
-                region: String::from("eu-west-1"),
+                region: Some(String::from("eu-west-1")),
+                endpoint: Some(String::from("omar")),
                 profile: Some(String::from("default")),
                 access_key: Some(String::from("pippo")),
                 secret_access_key: Some(String::from("pluto")),
+                new_path_style: Some(true),
             }),
         };
         let params = FileTransferParams::from(bookmark);
         assert_eq!(params.protocol, FileTransferProtocol::AwsS3);
         let gparams = params.params.s3_params().unwrap();
         assert_eq!(gparams.bucket_name.as_str(), "veeso");
-        assert_eq!(gparams.region.as_str(), "eu-west-1");
+        assert_eq!(gparams.region.as_deref().unwrap(), "eu-west-1");
+        assert_eq!(gparams.endpoint.as_deref().unwrap(), "omar");
         assert_eq!(gparams.profile.as_deref().unwrap(), "default");
         assert_eq!(gparams.access_key.as_deref().unwrap(), "pippo");
         assert_eq!(gparams.secret_access_key.as_deref().unwrap(), "pluto");
+        assert_eq!(gparams.new_path_style, true);
     }
 }

src/config/serialization.rs

@@ -419,10 +419,12 @@ mod tests {
         assert_eq!(host.protocol, FileTransferProtocol::AwsS3);
         let s3 = host.s3.as_ref().unwrap();
         assert_eq!(s3.bucket.as_str(), "veeso");
-        assert_eq!(s3.region.as_str(), "eu-west-1");
+        assert_eq!(s3.region.as_deref().unwrap(), "eu-west-1");
         assert_eq!(s3.profile.as_deref().unwrap(), "default");
+        assert_eq!(s3.endpoint.as_deref().unwrap(), "http://localhost:9000");
         assert_eq!(s3.access_key.as_deref().unwrap(), "pippo");
         assert_eq!(s3.secret_access_key.as_deref().unwrap(), "pluto");
+        assert_eq!(s3.new_path_style.unwrap(), true);
     }
 
     #[test]
@@ -470,10 +472,12 @@ mod tests {
                 password: None,
                 s3: Some(S3Params {
                     bucket: "veeso".to_string(),
-                    region: "eu-west-1".to_string(),
+                    region: Some("eu-west-1".to_string()),
+                    endpoint: None,
                     profile: None,
                     access_key: None,
                     secret_access_key: None,
+                    new_path_style: None,
                 }),
             },
         );
@@ -534,9 +538,11 @@ mod tests {
         [bookmarks.my-bucket.s3]
         bucket = "veeso"
         region = "eu-west-1"
+        endpoint = "http://localhost:9000"
         profile = "default"
         access_key = "pippo"
         secret_access_key = "pluto"
+        new_path_style = true
 
         [recents]
         ISO20201215T094000Z = { address = "172.16.104.10", port = 22, protocol = "SCP", username = "root" }

src/filetransfer/builder.rs

@@ -73,10 +73,16 @@ impl Builder {
 
     /// Build aws s3 client from parameters
     fn aws_s3_client(params: AwsS3Params) -> AwsS3Fs {
-        let mut client = AwsS3Fs::new(params.bucket_name, params.region);
+        let mut client = AwsS3Fs::new(params.bucket_name).new_path_style(params.new_path_style);
+        if let Some(region) = params.region {
+            client = client.region(region);
+        }
         if let Some(profile) = params.profile {
             client = client.profile(profile);
         }
+        if let Some(endpoint) = params.endpoint {
+            client = client.endpoint(endpoint);
+        }
         if let Some(access_key) = params.access_key {
             client = client.access_key(access_key);
         }
@@ -151,7 +157,9 @@ mod test {
     #[test]
     fn should_build_aws_s3_fs() {
         let params = ProtocolParams::AwsS3(
-            AwsS3Params::new("omar", "eu-west-1", Some("test"))
+            AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))
+                .endpoint(Some("http://localhost:9000"))
+                .new_path_style(true)
                 .access_key(Some("pippo"))
                 .secret_access_key(Some("pluto"))
                 .security_token(Some("omar"))

src/filetransfer/params.rs

@@ -59,12 +59,14 @@ pub struct GenericProtocolParams {
 #[derive(Debug, Clone)]
 pub struct AwsS3Params {
     pub bucket_name: String,
-    pub region: String,
+    pub region: Option<String>,
+    pub endpoint: Option<String>,
     pub profile: Option<String>,
     pub access_key: Option<String>,
     pub secret_access_key: Option<String>,
     pub security_token: Option<String>,
     pub session_token: Option<String>,
+    pub new_path_style: bool,
 }
 
 impl FileTransferParams {
@@ -167,18 +169,26 @@ impl GenericProtocolParams {
 
 impl AwsS3Params {
     /// Instantiates a new `AwsS3Params` struct
-    pub fn new<S: AsRef<str>>(bucket: S, region: S, profile: Option<S>) -> Self {
+    pub fn new<S: AsRef<str>>(bucket: S, region: Option<S>, profile: Option<S>) -> Self {
         Self {
             bucket_name: bucket.as_ref().to_string(),
-            region: region.as_ref().to_string(),
+            region: region.map(|x| x.as_ref().to_string()),
             profile: profile.map(|x| x.as_ref().to_string()),
+            endpoint: None,
             access_key: None,
             secret_access_key: None,
             security_token: None,
             session_token: None,
+            new_path_style: false,
         }
     }
 
+    /// Construct aws s3 params with specified endpoint
+    pub fn endpoint<S: AsRef<str>>(mut self, endpoint: Option<S>) -> Self {
+        self.endpoint = endpoint.map(|x| x.as_ref().to_string());
+        self
+    }
+
     /// Construct aws s3 params with provided access key
     pub fn access_key<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
         self.access_key = key.map(|x| x.as_ref().to_string());
@@ -202,6 +212,12 @@ impl AwsS3Params {
         self.session_token = key.map(|x| x.as_ref().to_string());
         self
     }
+
+    /// Specify new path style when constructing aws s3 params
+    pub fn new_path_style(mut self, new_path_style: bool) -> Self {
+        self.new_path_style = new_path_style;
+        self
+    }
 }
 
 #[cfg(test)]
@@ -240,35 +256,42 @@ mod test {
 
     #[test]
     fn should_init_aws_s3_params() {
-        let params: AwsS3Params = AwsS3Params::new("omar", "eu-west-1", Some("test"));
+        let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test"));
         assert_eq!(params.bucket_name.as_str(), "omar");
-        assert_eq!(params.region.as_str(), "eu-west-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
         assert_eq!(params.profile.as_deref().unwrap(), "test");
+        assert!(params.endpoint.is_none());
         assert!(params.access_key.is_none());
         assert!(params.secret_access_key.is_none());
         assert!(params.security_token.is_none());
         assert!(params.session_token.is_none());
+        assert_eq!(params.new_path_style, false);
     }
 
     #[test]
     fn should_init_aws_s3_params_with_optionals() {
-        let params: AwsS3Params = AwsS3Params::new("omar", "eu-west-1", Some("test"))
+        let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))
+            .endpoint(Some("http://omar.it"))
             .access_key(Some("pippo"))
             .secret_access_key(Some("pluto"))
             .security_token(Some("omar"))
-            .session_token(Some("gerry-scotti"));
+            .session_token(Some("gerry-scotti"))
+            .new_path_style(true);
         assert_eq!(params.bucket_name.as_str(), "omar");
-        assert_eq!(params.region.as_str(), "eu-west-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
         assert_eq!(params.profile.as_deref().unwrap(), "test");
+        assert_eq!(params.endpoint.as_deref().unwrap(), "http://omar.it");
         assert_eq!(params.access_key.as_deref().unwrap(), "pippo");
         assert_eq!(params.secret_access_key.as_deref().unwrap(), "pluto");
         assert_eq!(params.security_token.as_deref().unwrap(), "omar");
         assert_eq!(params.session_token.as_deref().unwrap(), "gerry-scotti");
+        assert_eq!(params.new_path_style, true);
     }
 
     #[test]
     fn references() {
-        let mut params = ProtocolParams::AwsS3(AwsS3Params::new("omar", "eu-west-1", Some("test")));
+        let mut params =
+            ProtocolParams::AwsS3(AwsS3Params::new("omar", Some("eu-west-1"), Some("test")));
         assert!(params.s3_params().is_some());
         assert!(params.generic_params().is_none());
         assert!(params.mut_generic_params().is_none());

src/system/bookmarks_client.rs

@@ -506,7 +506,7 @@ mod tests {
         assert_eq!(params.profile.as_deref().unwrap(), "test");
         assert_eq!(params.secret_access_key.as_deref().unwrap(), "pluto");
         assert_eq!(params.bucket_name.as_str(), "omar");
-        assert_eq!(params.region.as_str(), "eu-west-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
     }
 
     #[test]
@@ -524,7 +524,7 @@ mod tests {
         let params = bookmark.params.s3_params().unwrap();
         assert_eq!(params.profile.as_deref().unwrap(), "test");
         assert_eq!(params.bucket_name.as_str(), "omar");
-        assert_eq!(params.region.as_str(), "eu-west-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
         // secrets
         assert_eq!(params.access_key, None);
         assert_eq!(params.secret_access_key, None);
@@ -546,7 +546,7 @@ mod tests {
         let params = bookmark.params.s3_params().unwrap();
         assert_eq!(params.profile.as_deref().unwrap(), "test");
         assert_eq!(params.bucket_name.as_str(), "omar");
-        assert_eq!(params.region.as_str(), "eu-west-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
         // secrets
         assert_eq!(params.access_key, None);
         assert_eq!(params.secret_access_key, None);
@@ -849,7 +849,9 @@ mod tests {
         FileTransferParams::new(
             FileTransferProtocol::AwsS3,
             ProtocolParams::AwsS3(
-                AwsS3Params::new("omar", "eu-west-1", Some("test"))
+                AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))
+                    .endpoint(Some("http://localhost:9000"))
+                    .new_path_style(false)
                     .access_key(Some("pippo"))
                     .secret_access_key(Some("pluto"))
                     .security_token(Some("omar"))

src/ui/activities/auth/bookmarks.rs

@@ -227,11 +227,12 @@ impl AuthActivity {
 
     fn load_bookmark_s3_into_gui(&mut self, params: AwsS3Params) {
         self.mount_s3_bucket(params.bucket_name.as_str());
-        self.mount_s3_region(params.region.as_str());
+        self.mount_s3_region(params.region.as_deref().unwrap_or(""));
         self.mount_s3_profile(params.profile.as_deref().unwrap_or(""));
         self.mount_s3_access_key(params.access_key.as_deref().unwrap_or(""));
         self.mount_s3_secret_access_key(params.secret_access_key.as_deref().unwrap_or(""));
         self.mount_s3_security_token(params.security_token.as_deref().unwrap_or(""));
         self.mount_s3_session_token(params.session_token.as_deref().unwrap_or(""));
+        // TODO: add mount
     }
 }

src/ui/activities/auth/misc.rs

@@ -88,9 +88,6 @@ impl AuthActivity {
         if params.bucket_name.is_empty() {
             return Err("Invalid bucket");
         }
-        if params.region.is_empty() {
-            return Err("Invalid region");
-        }
         Ok(FileTransferParams {
             protocol: FileTransferProtocol::AwsS3,
             params: ProtocolParams::AwsS3(params),

src/ui/activities/auth/view.rs

@@ -726,12 +726,13 @@ impl AuthActivity {
     /// Collect s3 input values from view
     pub(super) fn get_s3_params_input(&self) -> AwsS3Params {
         let bucket: String = self.get_input_s3_bucket();
-        let region: String = self.get_input_s3_region();
+        let region: Option<String> = self.get_input_s3_region();
         let profile: Option<String> = self.get_input_s3_profile();
         let access_key = self.get_input_s3_access_key();
         let secret_access_key = self.get_input_s3_secret_access_key();
         let security_token = self.get_input_s3_security_token();
         let session_token = self.get_input_s3_session_token();
+        // TODO: collect
         AwsS3Params::new(bucket, region, profile)
             .access_key(access_key)
             .secret_access_key(secret_access_key)
@@ -777,10 +778,10 @@ impl AuthActivity {
         }
     }
 
-    pub(super) fn get_input_s3_region(&self) -> String {
+    pub(super) fn get_input_s3_region(&self) -> Option<String> {
         match self.app.state(&Id::S3Region) {
-            Ok(State::One(StateValue::String(x))) => x,
+            Ok(State::One(StateValue::String(x))) if !x.is_empty() => Some(x),
-            _ => String::new(),
+            _ => None,
         }
     }
 
@@ -863,8 +864,12 @@ impl AuthActivity {
                     None => String::default(),
                 };
                 format!(
-                    "{}://{} ({}) {}",
+                    "{}://{}{} ({}) {}",
-                    protocol, s3.bucket_name, s3.region, profile
+                    protocol,
+                    s3.endpoint.unwrap_or_default(),
+                    s3.bucket_name,
+                    s3.region.as_deref().unwrap_or("custom"),
+                    profile
                 )
             }
             ProtocolParams::Generic(params) => {

src/ui/activities/filetransfer/misc.rs

@@ -147,8 +147,10 @@ impl FileTransferActivity {
             }
             ProtocolParams::AwsS3(params) => {
                 info!(
-                    "Client is not connected to remote; connecting to {} ({})",
+                    "Client is not connected to remote; connecting to {}{} ({})",
-                    params.bucket_name, params.region
+                    params.endpoint.as_deref().unwrap_or(""),
+                    params.bucket_name,
+                    params.region.as_deref().unwrap_or("custom")
                 );
                 format!("Connecting to {}…", params.bucket_name)
             }

src/utils/parser.rs

@@ -243,7 +243,7 @@ fn parse_s3_remote_opt(s: &str) -> Result<FileTransferParams, String> {
                 groups.get(4).map(|group| PathBuf::from(group.as_str()));
             Ok(FileTransferParams::new(
                 FileTransferProtocol::AwsS3,
-                ProtocolParams::AwsS3(AwsS3Params::new(bucket, region, profile)),
+                ProtocolParams::AwsS3(AwsS3Params::new(bucket, Some(region), profile)),
             )
             .entry_directory(entry_directory))
         }
@@ -500,7 +500,7 @@ mod tests {
         assert_eq!(result.protocol, FileTransferProtocol::AwsS3);
         assert_eq!(result.entry_directory, None);
         assert_eq!(params.bucket_name.as_str(), "mybucket");
-        assert_eq!(params.region.as_str(), "eu-central-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-central-1");
         assert_eq!(params.profile, None);
         // With profile
         let result: FileTransferParams =
@@ -511,7 +511,7 @@ mod tests {
         assert_eq!(result.protocol, FileTransferProtocol::AwsS3);
         assert_eq!(result.entry_directory, None);
         assert_eq!(params.bucket_name.as_str(), "mybucket");
-        assert_eq!(params.region.as_str(), "eu-central-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-central-1");
         assert_eq!(params.profile.as_deref(), Some("default"));
         // With wrkdir only
         let result: FileTransferParams =
@@ -522,7 +522,7 @@ mod tests {
         assert_eq!(result.protocol, FileTransferProtocol::AwsS3);
         assert_eq!(result.entry_directory, Some(PathBuf::from("/foobar")));
         assert_eq!(params.bucket_name.as_str(), "mybucket");
-        assert_eq!(params.region.as_str(), "eu-central-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-central-1");
         assert_eq!(params.profile, None);
         // With all arguments
         let result: FileTransferParams =
@@ -533,7 +533,7 @@ mod tests {
         assert_eq!(result.protocol, FileTransferProtocol::AwsS3);
         assert_eq!(result.entry_directory, Some(PathBuf::from("/foobar")));
         assert_eq!(params.bucket_name.as_str(), "mybucket");
-        assert_eq!(params.region.as_str(), "eu-central-1");
+        assert_eq!(params.region.as_deref().unwrap(), "eu-central-1");
         assert_eq!(params.profile.as_deref(), Some("default"));
         // -- bad args
         assert!(parse_remote_opt(&String::from("s3://mybucket:default:/foobar")).is_err());