feat: kube protocol support (#267)

This commit is contained in:
Christian Visintin
2024-07-17 11:59:30 +02:00
committed by GitHub
parent cf529c1678
commit f757336d75
50 changed files with 1863 additions and 418 deletions

View File

@@ -3,10 +3,12 @@
//! Remotefs client builder
use std::path::PathBuf;
use std::sync::Arc;
use remotefs::RemoteFs;
use remotefs_aws_s3::AwsS3Fs;
use remotefs_ftp::FtpFs;
use remotefs_kube::KubeFs;
#[cfg(smb_unix)]
use remotefs_smb::SmbOptions;
#[cfg(smb)]
@@ -14,11 +16,11 @@ use remotefs_smb::{SmbCredentials, SmbFs};
use remotefs_ssh::{ScpFs, SftpFs, SshAgentIdentity, SshConfigParseRule, SshOpts};
use remotefs_webdav::WebDAVFs;
use super::params::WebDAVProtocolParams;
#[cfg(not(smb))]
use super::params::{AwsS3Params, GenericProtocolParams};
#[cfg(smb)]
use super::params::{AwsS3Params, GenericProtocolParams, SmbParams};
use super::params::{KubeProtocolParams, WebDAVProtocolParams};
use super::{FileTransferProtocol, ProtocolParams};
use crate::system::config_client::ConfigClient;
use crate::system::sshkey_storage::SshKeyStorage;
@@ -43,6 +45,9 @@ impl Builder {
(FileTransferProtocol::Ftp(secure), ProtocolParams::Generic(params)) => {
Box::new(Self::ftp_client(params, secure))
}
(FileTransferProtocol::Kube, ProtocolParams::Kube(params)) => {
Box::new(Self::kube_client(params))
}
(FileTransferProtocol::Scp, ProtocolParams::Generic(params)) => {
Box::new(Self::scp_client(params, config_client))
}
@@ -105,6 +110,23 @@ impl Builder {
client
}
/// Build kube client
fn kube_client(params: KubeProtocolParams) -> KubeFs {
let rt = Arc::new(
tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.enable_all()
.build()
.expect("Unable to create tokio runtime"),
);
let kube_fs = KubeFs::new(&params.pod, &params.container, &rt);
if let Some(config) = params.config() {
kube_fs.config(config)
} else {
kube_fs
}
}
/// Build scp client
fn scp_client(params: GenericProtocolParams, config_client: &ConfigClient) -> ScpFs {
Self::build_ssh_opts(params, config_client).into()
@@ -256,6 +278,21 @@ mod test {
let _ = Builder::build(FileTransferProtocol::Ftp(true), params, &config_client);
}
#[test]
fn test_should_build_kube_fs() {
let params = ProtocolParams::Kube(KubeProtocolParams {
pod: "pod".to_string(),
container: "container".to_string(),
namespace: Some("namespace".to_string()),
cluster_url: Some("cluster_url".to_string()),
username: Some("username".to_string()),
client_cert: Some("client_cert".to_string()),
client_key: Some("client_key".to_string()),
});
let config_client = get_config_client();
let _ = Builder::build(FileTransferProtocol::Kube, params, &config_client);
}
#[test]
fn should_build_scp_fs() {
let params = ProtocolParams::Generic(

View File

@@ -15,6 +15,7 @@ pub use params::{FileTransferParams, ProtocolParams};
pub enum FileTransferProtocol {
AwsS3,
Ftp(bool), // Bool is for secure (true => ftps)
Kube,
Scp,
Sftp,
Smb,
@@ -34,6 +35,7 @@ impl std::fmt::Display for FileTransferProtocol {
true => "FTPS",
false => "FTP",
},
FileTransferProtocol::Kube => "KUBE",
FileTransferProtocol::Scp => "SCP",
FileTransferProtocol::Sftp => "SFTP",
FileTransferProtocol::Smb => "SMB",
@@ -49,6 +51,7 @@ impl std::str::FromStr for FileTransferProtocol {
match s.to_ascii_uppercase().as_str() {
"FTP" => Ok(FileTransferProtocol::Ftp(false)),
"FTPS" => Ok(FileTransferProtocol::Ftp(true)),
"KUBE" => Ok(FileTransferProtocol::Kube),
"S3" => Ok(FileTransferProtocol::AwsS3),
"SCP" => Ok(FileTransferProtocol::Scp),
"SFTP" => Ok(FileTransferProtocol::Sftp),
@@ -114,6 +117,14 @@ mod tests {
FileTransferProtocol::from_str("scp").ok().unwrap(),
FileTransferProtocol::Scp
);
assert_eq!(
FileTransferProtocol::from_str("kube").ok().unwrap(),
FileTransferProtocol::Kube
);
assert_eq!(
FileTransferProtocol::from_str("KUBE").ok().unwrap(),
FileTransferProtocol::Kube
);
assert_eq!(
FileTransferProtocol::from_str("SMB").ok().unwrap(),
FileTransferProtocol::Smb
@@ -153,5 +164,6 @@ mod tests {
FileTransferProtocol::WebDAV.to_string(),
String::from("WEBDAV")
);
assert_eq!(FileTransferProtocol::Kube.to_string(), String::from("KUBE"));
}
}

View File

@@ -2,8 +2,17 @@
//!
//! file transfer parameters
mod aws_s3;
mod kube;
mod smb;
mod webdav;
use std::path::{Path, PathBuf};
pub use self::aws_s3::AwsS3Params;
pub use self::kube::KubeProtocolParams;
pub use self::smb::SmbParams;
pub use self::webdav::WebDAVProtocolParams;
use super::FileTransferProtocol;
/// Holds connection parameters for file transfers
@@ -20,6 +29,7 @@ pub struct FileTransferParams {
pub enum ProtocolParams {
Generic(GenericProtocolParams),
AwsS3(AwsS3Params),
Kube(KubeProtocolParams),
Smb(SmbParams),
WebDAV(WebDAVProtocolParams),
}
@@ -33,33 +43,6 @@ pub struct GenericProtocolParams {
pub password: Option<String>,
}
/// Connection parameters for AWS S3 protocol
#[derive(Debug, Clone)]
pub struct AwsS3Params {
pub bucket_name: String,
pub region: Option<String>,
pub endpoint: Option<String>,
pub profile: Option<String>,
pub access_key: Option<String>,
pub secret_access_key: Option<String>,
pub security_token: Option<String>,
pub session_token: Option<String>,
pub new_path_style: bool,
}
/// Connection parameters for SMB protocol
#[derive(Debug, Clone)]
pub struct SmbParams {
pub address: String,
#[cfg(unix)]
pub port: u16,
pub share: String,
pub username: Option<String>,
pub password: Option<String>,
#[cfg(unix)]
pub workgroup: Option<String>,
}
impl FileTransferParams {
/// Instantiates a new `FileTransferParams`
pub fn new(protocol: FileTransferProtocol, params: ProtocolParams) -> Self {
@@ -89,6 +72,7 @@ impl FileTransferParams {
match &self.params {
ProtocolParams::AwsS3(params) => params.password_missing(),
ProtocolParams::Generic(params) => params.password_missing(),
ProtocolParams::Kube(params) => params.password_missing(),
ProtocolParams::Smb(params) => params.password_missing(),
ProtocolParams::WebDAV(params) => params.password_missing(),
}
@@ -99,30 +83,13 @@ impl FileTransferParams {
match &mut self.params {
ProtocolParams::AwsS3(params) => params.set_default_secret(secret),
ProtocolParams::Generic(params) => params.set_default_secret(secret),
ProtocolParams::Kube(params) => params.set_default_secret(secret),
ProtocolParams::Smb(params) => params.set_default_secret(secret),
ProtocolParams::WebDAV(params) => params.set_default_secret(secret),
}
}
}
/// Protocol params used by WebDAV
#[derive(Debug, Clone)]
pub struct WebDAVProtocolParams {
pub uri: String,
pub username: String,
pub password: String,
}
impl WebDAVProtocolParams {
fn set_default_secret(&mut self, secret: String) {
self.password = secret;
}
fn password_missing(&self) -> bool {
self.password.is_empty()
}
}
impl Default for FileTransferParams {
fn default() -> Self {
Self::new(FileTransferProtocol::Sftp, ProtocolParams::default())
@@ -162,6 +129,15 @@ impl ProtocolParams {
}
}
#[cfg(test)]
/// Retrieve Kube params parameters if any
pub fn kube_params(&self) -> Option<&KubeProtocolParams> {
match self {
ProtocolParams::Kube(params) => Some(params),
_ => None,
}
}
#[cfg(test)]
/// Retrieve SMB parameters if any
pub fn smb_params(&self) -> Option<&SmbParams> {
@@ -231,127 +207,6 @@ impl GenericProtocolParams {
}
}
// -- S3 params
impl AwsS3Params {
/// Instantiates a new `AwsS3Params` struct
pub fn new<S: AsRef<str>>(bucket: S, region: Option<S>, profile: Option<S>) -> Self {
Self {
bucket_name: bucket.as_ref().to_string(),
region: region.map(|x| x.as_ref().to_string()),
profile: profile.map(|x| x.as_ref().to_string()),
endpoint: None,
access_key: None,
secret_access_key: None,
security_token: None,
session_token: None,
new_path_style: false,
}
}
/// Construct aws s3 params with specified endpoint
pub fn endpoint<S: AsRef<str>>(mut self, endpoint: Option<S>) -> Self {
self.endpoint = endpoint.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided access key
pub fn access_key<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.access_key = key.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided secret_access_key
pub fn secret_access_key<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.secret_access_key = key.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided security_token
pub fn security_token<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.security_token = key.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided session_token
pub fn session_token<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.session_token = key.map(|x| x.as_ref().to_string());
self
}
/// Specify new path style when constructing aws s3 params
pub fn new_path_style(mut self, new_path_style: bool) -> Self {
self.new_path_style = new_path_style;
self
}
/// Returns whether a password is supposed to be required for this protocol params.
/// The result true is returned ONLY if the supposed secret is MISSING!!!
pub fn password_missing(&self) -> bool {
self.secret_access_key.is_none() && self.security_token.is_none()
}
/// Set password
pub fn set_default_secret(&mut self, secret: String) {
self.secret_access_key = Some(secret);
}
}
// -- SMB params
impl SmbParams {
/// Instantiates a new `AwsS3Params` struct
pub fn new<S: AsRef<str>>(address: S, share: S) -> Self {
Self {
address: address.as_ref().to_string(),
#[cfg(unix)]
port: 445,
share: share.as_ref().to_string(),
username: None,
password: None,
#[cfg(unix)]
workgroup: None,
}
}
#[cfg(unix)]
pub fn port(mut self, port: u16) -> Self {
self.port = port;
self
}
pub fn username(mut self, username: Option<impl ToString>) -> Self {
self.username = username.map(|x| x.to_string());
self
}
pub fn password(mut self, password: Option<impl ToString>) -> Self {
self.password = password.map(|x| x.to_string());
self
}
#[cfg(unix)]
pub fn workgroup(mut self, workgroup: Option<impl ToString>) -> Self {
self.workgroup = workgroup.map(|x| x.to_string());
self
}
/// Returns whether a password is supposed to be required for this protocol params.
/// The result true is returned ONLY if the supposed secret is MISSING!!!
pub fn password_missing(&self) -> bool {
self.password.is_none()
}
/// Set password
#[cfg(unix)]
pub fn set_default_secret(&mut self, secret: String) {
self.password = Some(secret);
}
#[cfg(windows)]
pub fn set_default_secret(&mut self, _secret: String) {}
}
#[cfg(test)]
mod test {
@@ -386,87 +241,6 @@ mod test {
assert!(params.password.is_none());
}
#[test]
fn should_init_aws_s3_params() {
let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test"));
assert_eq!(params.bucket_name.as_str(), "omar");
assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
assert_eq!(params.profile.as_deref().unwrap(), "test");
assert!(params.endpoint.is_none());
assert!(params.access_key.is_none());
assert!(params.secret_access_key.is_none());
assert!(params.security_token.is_none());
assert!(params.session_token.is_none());
assert_eq!(params.new_path_style, false);
}
#[test]
fn should_init_aws_s3_params_with_optionals() {
let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))
.endpoint(Some("http://omar.it"))
.access_key(Some("pippo"))
.secret_access_key(Some("pluto"))
.security_token(Some("omar"))
.session_token(Some("gerry-scotti"))
.new_path_style(true);
assert_eq!(params.bucket_name.as_str(), "omar");
assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
assert_eq!(params.profile.as_deref().unwrap(), "test");
assert_eq!(params.endpoint.as_deref().unwrap(), "http://omar.it");
assert_eq!(params.access_key.as_deref().unwrap(), "pippo");
assert_eq!(params.secret_access_key.as_deref().unwrap(), "pluto");
assert_eq!(params.security_token.as_deref().unwrap(), "omar");
assert_eq!(params.session_token.as_deref().unwrap(), "gerry-scotti");
assert_eq!(params.new_path_style, true);
}
#[test]
fn should_init_smb_params() {
let params = SmbParams::new("localhost", "temp");
assert_eq!(&params.address, "localhost");
#[cfg(unix)]
assert_eq!(params.port, 445);
assert_eq!(&params.share, "temp");
#[cfg(unix)]
assert!(params.username.is_none());
#[cfg(unix)]
assert!(params.password.is_none());
#[cfg(unix)]
assert!(params.workgroup.is_none());
}
#[test]
#[cfg(unix)]
fn should_init_smb_params_with_optionals() {
let params = SmbParams::new("localhost", "temp")
.port(3456)
.username(Some("foo"))
.password(Some("bar"))
.workgroup(Some("baz"));
assert_eq!(&params.address, "localhost");
assert_eq!(params.port, 3456);
assert_eq!(&params.share, "temp");
assert_eq!(params.username.as_deref().unwrap(), "foo");
assert_eq!(params.password.as_deref().unwrap(), "bar");
assert_eq!(params.workgroup.as_deref().unwrap(), "baz");
}
#[test]
#[cfg(windows)]
fn should_init_smb_params_with_optionals() {
let params = SmbParams::new("localhost", "temp")
.username(Some("foo"))
.password(Some("bar"));
assert_eq!(&params.address, "localhost");
assert_eq!(&params.share, "temp");
assert_eq!(params.username.as_deref().unwrap(), "foo");
assert_eq!(params.password.as_deref().unwrap(), "bar");
}
#[test]
fn references() {
let mut params =

View File

@@ -0,0 +1,121 @@
/// Connection parameters for AWS S3 protocol
#[derive(Debug, Clone)]
pub struct AwsS3Params {
pub bucket_name: String,
pub region: Option<String>,
pub endpoint: Option<String>,
pub profile: Option<String>,
pub access_key: Option<String>,
pub secret_access_key: Option<String>,
pub security_token: Option<String>,
pub session_token: Option<String>,
pub new_path_style: bool,
}
// -- S3 params
impl AwsS3Params {
/// Instantiates a new `AwsS3Params` struct
pub fn new<S: AsRef<str>>(bucket: S, region: Option<S>, profile: Option<S>) -> Self {
Self {
bucket_name: bucket.as_ref().to_string(),
region: region.map(|x| x.as_ref().to_string()),
profile: profile.map(|x| x.as_ref().to_string()),
endpoint: None,
access_key: None,
secret_access_key: None,
security_token: None,
session_token: None,
new_path_style: false,
}
}
/// Construct aws s3 params with specified endpoint
pub fn endpoint<S: AsRef<str>>(mut self, endpoint: Option<S>) -> Self {
self.endpoint = endpoint.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided access key
pub fn access_key<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.access_key = key.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided secret_access_key
pub fn secret_access_key<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.secret_access_key = key.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided security_token
pub fn security_token<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.security_token = key.map(|x| x.as_ref().to_string());
self
}
/// Construct aws s3 params with provided session_token
pub fn session_token<S: AsRef<str>>(mut self, key: Option<S>) -> Self {
self.session_token = key.map(|x| x.as_ref().to_string());
self
}
/// Specify new path style when constructing aws s3 params
pub fn new_path_style(mut self, new_path_style: bool) -> Self {
self.new_path_style = new_path_style;
self
}
/// Returns whether a password is supposed to be required for this protocol params.
/// The result true is returned ONLY if the supposed secret is MISSING!!!
pub fn password_missing(&self) -> bool {
self.secret_access_key.is_none() && self.security_token.is_none()
}
/// Set password
pub fn set_default_secret(&mut self, secret: String) {
self.secret_access_key = Some(secret);
}
}
#[cfg(test)]
mod test {
use pretty_assertions::assert_eq;
use super::*;
#[test]
fn should_init_aws_s3_params() {
let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test"));
assert_eq!(params.bucket_name.as_str(), "omar");
assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
assert_eq!(params.profile.as_deref().unwrap(), "test");
assert!(params.endpoint.is_none());
assert!(params.access_key.is_none());
assert!(params.secret_access_key.is_none());
assert!(params.security_token.is_none());
assert!(params.session_token.is_none());
assert_eq!(params.new_path_style, false);
}
#[test]
fn should_init_aws_s3_params_with_optionals() {
let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))
.endpoint(Some("http://omar.it"))
.access_key(Some("pippo"))
.secret_access_key(Some("pluto"))
.security_token(Some("omar"))
.session_token(Some("gerry-scotti"))
.new_path_style(true);
assert_eq!(params.bucket_name.as_str(), "omar");
assert_eq!(params.region.as_deref().unwrap(), "eu-west-1");
assert_eq!(params.profile.as_deref().unwrap(), "test");
assert_eq!(params.endpoint.as_deref().unwrap(), "http://omar.it");
assert_eq!(params.access_key.as_deref().unwrap(), "pippo");
assert_eq!(params.secret_access_key.as_deref().unwrap(), "pluto");
assert_eq!(params.security_token.as_deref().unwrap(), "omar");
assert_eq!(params.session_token.as_deref().unwrap(), "gerry-scotti");
assert_eq!(params.new_path_style, true);
}
}

View File

@@ -0,0 +1,37 @@
use remotefs_kube::Config;
/// Protocol params used by WebDAV
#[derive(Debug, Clone)]
pub struct KubeProtocolParams {
pub pod: String,
pub container: String,
pub namespace: Option<String>,
pub cluster_url: Option<String>,
pub username: Option<String>,
pub client_cert: Option<String>,
pub client_key: Option<String>,
}
impl KubeProtocolParams {
pub fn set_default_secret(&mut self, _secret: String) {}
pub fn password_missing(&self) -> bool {
false
}
pub fn config(self) -> Option<Config> {
if let Some(cluster_url) = self.cluster_url {
let mut config = Config::new(cluster_url.parse().unwrap_or_default());
config.auth_info.username = self.username;
config.auth_info.client_certificate = self.client_cert;
config.auth_info.client_key = self.client_key;
if let Some(namespace) = self.namespace {
config.default_namespace = namespace;
}
Some(config)
} else {
None
}
}
}

View File

@@ -0,0 +1,122 @@
/// Connection parameters for SMB protocol
#[derive(Debug, Clone)]
pub struct SmbParams {
pub address: String,
#[cfg(unix)]
pub port: u16,
pub share: String,
pub username: Option<String>,
pub password: Option<String>,
#[cfg(unix)]
pub workgroup: Option<String>,
}
// -- SMB params
impl SmbParams {
/// Instantiates a new `AwsS3Params` struct
pub fn new<S: AsRef<str>>(address: S, share: S) -> Self {
Self {
address: address.as_ref().to_string(),
#[cfg(unix)]
port: 445,
share: share.as_ref().to_string(),
username: None,
password: None,
#[cfg(unix)]
workgroup: None,
}
}
#[cfg(unix)]
pub fn port(mut self, port: u16) -> Self {
self.port = port;
self
}
pub fn username(mut self, username: Option<impl ToString>) -> Self {
self.username = username.map(|x| x.to_string());
self
}
pub fn password(mut self, password: Option<impl ToString>) -> Self {
self.password = password.map(|x| x.to_string());
self
}
#[cfg(unix)]
pub fn workgroup(mut self, workgroup: Option<impl ToString>) -> Self {
self.workgroup = workgroup.map(|x| x.to_string());
self
}
/// Returns whether a password is supposed to be required for this protocol params.
/// The result true is returned ONLY if the supposed secret is MISSING!!!
pub fn password_missing(&self) -> bool {
self.password.is_none()
}
/// Set password
#[cfg(unix)]
pub fn set_default_secret(&mut self, secret: String) {
self.password = Some(secret);
}
#[cfg(windows)]
pub fn set_default_secret(&mut self, _secret: String) {}
}
#[cfg(test)]
mod test {
use pretty_assertions::assert_eq;
use super::*;
#[test]
fn should_init_smb_params() {
let params = SmbParams::new("localhost", "temp");
assert_eq!(&params.address, "localhost");
#[cfg(unix)]
assert_eq!(params.port, 445);
assert_eq!(&params.share, "temp");
#[cfg(unix)]
assert!(params.username.is_none());
#[cfg(unix)]
assert!(params.password.is_none());
#[cfg(unix)]
assert!(params.workgroup.is_none());
}
#[test]
#[cfg(unix)]
fn should_init_smb_params_with_optionals() {
let params = SmbParams::new("localhost", "temp")
.port(3456)
.username(Some("foo"))
.password(Some("bar"))
.workgroup(Some("baz"));
assert_eq!(&params.address, "localhost");
assert_eq!(params.port, 3456);
assert_eq!(&params.share, "temp");
assert_eq!(params.username.as_deref().unwrap(), "foo");
assert_eq!(params.password.as_deref().unwrap(), "bar");
assert_eq!(params.workgroup.as_deref().unwrap(), "baz");
}
#[test]
#[cfg(windows)]
fn should_init_smb_params_with_optionals() {
let params = SmbParams::new("localhost", "temp")
.username(Some("foo"))
.password(Some("bar"));
assert_eq!(&params.address, "localhost");
assert_eq!(&params.share, "temp");
assert_eq!(params.username.as_deref().unwrap(), "foo");
assert_eq!(params.password.as_deref().unwrap(), "bar");
}
}

View File

@@ -0,0 +1,17 @@
/// Protocol params used by WebDAV
#[derive(Debug, Clone)]
pub struct WebDAVProtocolParams {
pub uri: String,
pub username: String,
pub password: String,
}
impl WebDAVProtocolParams {
pub fn set_default_secret(&mut self, secret: String) {
self.password = secret;
}
pub fn password_missing(&self) -> bool {
self.password.is_empty()
}
}