vh/termscp
1
0
Fork 0
mirror of https://github.com/veeso/termscp.git synced 2025-12-07 09:36:00 -08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Aws s3 support

Browse Source
This commit is contained in:
veeso
2021-08-26 11:24:13 +02:00
parent f31f58aa79
commit 1d09095ab9
37 changed files with 3458 additions and 973 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
src/filetransfer/mod.rs
View File

@@ -28,27 +28,29 @@
// locals
use crate::fs::{FsEntry, FsFile};
// ext
use std::io::{Read, Write};
use std::fs::File;
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use thiserror::Error;
use wildmatch::WildMatch;
// exports
pub mod ftp_transfer;
pub mod params;
pub mod scp_transfer;
pub mod sftp_transfer;
mod transfer;
pub use params::FileTransferParams;
// -- export types
pub use params::{FileTransferParams, ProtocolParams};
pub use transfer::{FtpFileTransfer, S3FileTransfer, ScpFileTransfer, SftpFileTransfer};
/// ## FileTransferProtocol
///
/// This enum defines the different transfer protocol available in termscp
#[derive(PartialEq, Debug, std::clone::Clone, Copy)]
#[derive(PartialEq, Debug, Clone, Copy)]
pub enum FileTransferProtocol {
Sftp,
Scp,
Ftp(bool), // Bool is for secure (true => ftps)
AwsS3,
}
/// ## FileTransferError
@@ -130,25 +132,16 @@ impl std::fmt::Display for FileTransferError {
/// ## FileTransfer
///
/// File transfer trait must be implemented by all the file transfers and defines the method used by a generic file transfer
pub trait FileTransfer {
/// ### connect
///
/// Connect to the remote server
/// Can return banner / welcome message on success
fn connect(
&mut self,
address: String,
port: u16,
username: Option<String>,
password: Option<String>,
) -> Result<Option<String>, FileTransferError>;
fn connect(&mut self, params: &ProtocolParams) -> Result<Option<String>, FileTransferError>;
/// ### disconnect
///
/// Disconnect from the remote server
fn disconnect(&mut self) -> Result<(), FileTransferError>;
/// ### is_connected
@@ -210,18 +203,28 @@ pub trait FileTransfer {
/// Send file to remote
/// File name is referred to the name of the file as it will be saved
/// Data contains the file data
/// Returns file and its size
/// Returns file and its size.
/// By default returns unsupported feature
fn send_file(
&mut self,
local: &FsFile,
file_name: &Path,
) -> Result<Box<dyn Write>, FileTransferError>;
_local: &FsFile,
_file_name: &Path,
) -> Result<Box<dyn Write>, FileTransferError> {
Err(FileTransferError::new(
FileTransferErrorType::UnsupportedFeature,
))
}
/// ### recv_file
///
/// Receive file from remote with provided name
/// Returns file and its size
fn recv_file(&mut self, file: &FsFile) -> Result<Box<dyn Read>, FileTransferError>;
/// By default returns unsupported feature
fn recv_file(&mut self, _file: &FsFile) -> Result<Box<dyn Read>, FileTransferError> {
Err(FileTransferError::new(
FileTransferErrorType::UnsupportedFeature,
))
}
/// ### on_sent
///
@@ -230,7 +233,10 @@ pub trait FileTransfer {
/// The purpose of this method is to finalize the connection with the peer when writing data.
/// This is necessary for some protocols such as FTP.
/// You must call this method each time you want to finalize the write of the remote file.
fn on_sent(&mut self, writable: Box<dyn Write>) -> Result<(), FileTransferError>;
/// By default this function returns already `Ok(())`
fn on_sent(&mut self, _writable: Box<dyn Write>) -> Result<(), FileTransferError> {
Ok(())
}
/// ### on_recv
///
@@ -239,7 +245,71 @@ pub trait FileTransfer {
/// The purpose of this method is to finalize the connection with the peer when reading data.
/// This mighe be necessary for some protocols.
/// You must call this method each time you want to finalize the read of the remote file.
fn on_recv(&mut self, readable: Box<dyn Read>) -> Result<(), FileTransferError>;
/// By default this function returns already `Ok(())`
fn on_recv(&mut self, _readable: Box<dyn Read>) -> Result<(), FileTransferError> {
Ok(())
}
/// ### send_file_wno_stream
///
/// Send a file to remote WITHOUT using streams.
/// This method SHOULD be implemented ONLY when streams are not supported by the current file transfer.
/// The developer implementing the filetransfer user should FIRST try with `send_file` followed by `on_sent`
/// If the function returns error kind() `UnsupportedFeature`, then he should call this function.
/// By default this function uses the streams function to copy content from reader to writer
fn send_file_wno_stream(
&mut self,
src: &FsFile,
dest: &Path,
mut reader: Box<dyn Read>,
) -> Result<(), FileTransferError> {
match self.is_connected() {
true => {
let mut stream = self.send_file(src, dest)?;
io::copy(&mut reader, &mut stream).map_err(|e| {
FileTransferError::new_ex(FileTransferErrorType::ProtocolError, e.to_string())
})?;
self.on_sent(stream)
}
false => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### recv_file_wno_stream
///
/// Receive a file from remote WITHOUT using streams.
/// This method SHOULD be implemented ONLY when streams are not supported by the current file transfer.
/// The developer implementing the filetransfer user should FIRST try with `send_file` followed by `on_sent`
/// If the function returns error kind() `UnsupportedFeature`, then he should call this function.
/// For safety reasons this function doesn't accept the `Write` trait, but the destination path.
/// By default this function uses the streams function to copy content from reader to writer
fn recv_file_wno_stream(&mut self, src: &FsFile, dest: &Path) -> Result<(), FileTransferError> {
match self.is_connected() {
true => {
let mut writer = File::create(dest).map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::FileCreateDenied,
format!("Could not open local file: {}", e),
)
})?;
let mut stream = self.recv_file(src)?;
io::copy(&mut stream, &mut writer)
.map(|_| ())
.map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::ProtocolError,
e.to_string(),
)
})?;
self.on_recv(stream)
}
false => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### find
///
@@ -314,6 +384,7 @@ impl std::string::ToString for FileTransferProtocol {
},
FileTransferProtocol::Scp => "SCP",
FileTransferProtocol::Sftp => "SFTP",
FileTransferProtocol::AwsS3 => "S3",
})
}
}
@@ -326,6 +397,7 @@ impl std::str::FromStr for FileTransferProtocol {
"FTPS" => Ok(FileTransferProtocol::Ftp(true)),
"SCP" => Ok(FileTransferProtocol::Scp),
"SFTP" => Ok(FileTransferProtocol::Sftp),
"S3" => Ok(FileTransferProtocol::AwsS3),
_ => Err(s.to_string()),
}
}
@@ -385,6 +457,14 @@ mod tests {
FileTransferProtocol::from_str("scp").ok().unwrap(),
FileTransferProtocol::Scp
);
assert_eq!(
FileTransferProtocol::from_str("S3").ok().unwrap(),
FileTransferProtocol::AwsS3
);
assert_eq!(
FileTransferProtocol::from_str("s3").ok().unwrap(),
FileTransferProtocol::AwsS3
);
// Error
assert!(FileTransferProtocol::from_str("dummy").is_err());
// To String
@@ -398,6 +478,7 @@ mod tests {
);
assert_eq!(FileTransferProtocol::Scp.to_string(), String::from("SCP"));
assert_eq!(FileTransferProtocol::Sftp.to_string(), String::from("SFTP"));
assert_eq!(FileTransferProtocol::AwsS3.to_string(), String::from("S3"));
}
#[test]

194
src/filetransfer/params.rs
View File

@@ -32,44 +32,132 @@ use std::path::{Path, PathBuf};
/// ### FileTransferParams
///
/// Holds connection parameters for file transfers
#[derive(Clone)]
#[derive(Debug, Clone)]
pub struct FileTransferParams {
pub protocol: FileTransferProtocol,
pub params: ProtocolParams,
pub entry_directory: Option<PathBuf>,
}
/// ## ProtocolParams
///
/// Container for protocol params
#[derive(Debug, Clone)]
pub enum ProtocolParams {
Generic(GenericProtocolParams),
AwsS3(AwsS3Params),
}
/// ## GenericProtocolParams
///
/// Protocol params used by most common protocols
#[derive(Debug, Clone)]
pub struct GenericProtocolParams {
pub address: String,
pub port: u16,
pub protocol: FileTransferProtocol,
pub username: Option<String>,
pub password: Option<String>,
pub entry_directory: Option<PathBuf>,
}
/// ## AwsS3Params
///
/// Connection parameters for AWS S3 protocol
#[derive(Debug, Clone)]
pub struct AwsS3Params {
pub bucket_name: String,
pub region: String,
pub profile: Option<String>,
}
impl FileTransferParams {
/// ### new
///
/// Instantiates a new `FileTransferParams`
pub fn new<S: AsRef<str>>(address: S) -> Self {
pub fn new(protocol: FileTransferProtocol, params: ProtocolParams) -> Self {
Self {
address: address.as_ref().to_string(),
port: 22,
protocol: FileTransferProtocol::Sftp,
username: None,
password: None,
protocol,
params,
entry_directory: None,
}
}
/// ### port
/// ### entry_directory
///
/// Set port for params
pub fn port(mut self, port: u16) -> Self {
self.port = port;
/// Set entry directory
pub fn entry_directory<P: AsRef<Path>>(mut self, dir: Option<P>) -> Self {
self.entry_directory = dir.map(|x| x.as_ref().to_path_buf());
self
}
}
impl Default for FileTransferParams {
fn default() -> Self {
Self::new(FileTransferProtocol::Sftp, ProtocolParams::default())
}
}
impl Default for ProtocolParams {
fn default() -> Self {
Self::Generic(GenericProtocolParams::default())
}
}
impl ProtocolParams {
/// ### generic_params
///
/// Retrieve generic parameters from protocol params if any
pub fn generic_params(&self) -> Option<&GenericProtocolParams> {
match self {
ProtocolParams::Generic(params) => Some(params),
_ => None,
}
}
pub fn mut_generic_params(&mut self) -> Option<&mut GenericProtocolParams> {
match self {
ProtocolParams::Generic(params) => Some(params),
_ => None,
}
}
/// ### s3_params
///
/// Retrieve AWS S3 parameters if any
pub fn s3_params(&self) -> Option<&AwsS3Params> {
match self {
ProtocolParams::AwsS3(params) => Some(params),
_ => None,
}
}
}
// -- Generic protocol params
impl Default for GenericProtocolParams {
fn default() -> Self {
Self {
address: "localhost".to_string(),
port: 22,
username: None,
password: None,
}
}
}
impl GenericProtocolParams {
/// ### address
///
/// Set address to params
pub fn address<S: AsRef<str>>(mut self, address: S) -> Self {
self.address = address.as_ref().to_string();
self
}
/// ### protocol
/// ### port
///
/// Set protocol for params
pub fn protocol(mut self, protocol: FileTransferProtocol) -> Self {
self.protocol = protocol;
/// Set port to params
pub fn port(mut self, port: u16) -> Self {
self.port = port;
self
}
@@ -88,19 +176,20 @@ impl FileTransferParams {
self.password = password.map(|x| x.as_ref().to_string());
self
}
/// ### entry_directory
///
/// Set entry directory
pub fn entry_directory<P: AsRef<Path>>(mut self, dir: Option<P>) -> Self {
self.entry_directory = dir.map(|x| x.as_ref().to_path_buf());
self
}
}
impl Default for FileTransferParams {
fn default() -> Self {
Self::new("localhost")
// -- S3 params
impl AwsS3Params {
/// ### new
///
/// Instantiates a new `AwsS3Params` struct
pub fn new<S: AsRef<str>>(bucket: S, region: S, profile: Option<S>) -> Self {
Self {
bucket_name: bucket.as_ref().to_string(),
region: region.as_ref().to_string(),
profile: profile.map(|x| x.as_ref().to_string()),
}
}
}
@@ -112,26 +201,49 @@ mod test {
#[test]
fn test_filetransfer_params() {
let params: FileTransferParams = FileTransferParams::new("test.rebex.net")
.port(2222)
.protocol(FileTransferProtocol::Scp)
.username(Some("omar"))
.password(Some("foobar"))
.entry_directory(Some(&Path::new("/tmp")));
assert_eq!(params.address.as_str(), "test.rebex.net");
assert_eq!(params.port, 2222);
let params: FileTransferParams =
FileTransferParams::new(FileTransferProtocol::Scp, ProtocolParams::default())
.entry_directory(Some(&Path::new("/tmp")));
assert_eq!(
params.params.generic_params().unwrap().address.as_str(),
"localhost"
);
assert_eq!(params.protocol, FileTransferProtocol::Scp);
assert_eq!(params.username.as_ref().unwrap(), "omar");
assert_eq!(params.password.as_ref().unwrap(), "foobar");
assert_eq!(
params.entry_directory.as_deref().unwrap(),
Path::new("/tmp")
);
}
#[test]
fn test_filetransfer_params_default() {
let params: FileTransferParams = FileTransferParams::default();
fn params_default() {
let params: GenericProtocolParams = ProtocolParams::default()
.generic_params()
.unwrap()
.to_owned();
assert_eq!(params.address.as_str(), "localhost");
assert_eq!(params.port, 22);
assert_eq!(params.protocol, FileTransferProtocol::Sftp);
assert!(params.username.is_none());
assert!(params.password.is_none());
}
#[test]
fn params_aws_s3() {
let params: AwsS3Params = AwsS3Params::new("omar", "eu-west-1", Some("test"));
assert_eq!(params.bucket_name.as_str(), "omar");
assert_eq!(params.region.as_str(), "eu-west-1");
assert_eq!(params.profile.as_deref().unwrap(), "test");
}
#[test]
fn references() {
let mut params = ProtocolParams::AwsS3(AwsS3Params::new("omar", "eu-west-1", Some("test")));
assert!(params.s3_params().is_some());
assert!(params.generic_params().is_none());
assert!(params.mut_generic_params().is_none());
let mut params = ProtocolParams::default();
assert!(params.s3_params().is_none());
assert!(params.generic_params().is_some());
assert!(params.mut_generic_params().is_some());
}
}

113
src/filetransfer/ftp_transfer.rs → src/filetransfer/transfer/ftp.rs
View File

@@ -1,4 +1,4 @@
//! ## Ftp_transfer
//! ## FTP transfer
//!
//! `ftp_transfer` is the module which provides the implementation for the FTP/FTPS file transfer
@@ -25,7 +25,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
use super::{FileTransfer, FileTransferError, FileTransferErrorType};
use super::{FileTransfer, FileTransferError, FileTransferErrorType, ProtocolParams};
use crate::fs::{FsDirectory, FsEntry, FsFile, UnixPex};
use crate::utils::fmt::shadow_password;
use crate::utils::path;
@@ -178,25 +178,24 @@ impl FileTransfer for FtpFileTransfer {
///
/// Connect to the remote server
fn connect(
&mut self,
address: String,
port: u16,
username: Option<String>,
password: Option<String>,
) -> Result<Option<String>, FileTransferError> {
// Get stream
info!("Connecting to {}:{}", address, port);
let mut stream: FtpStream = match FtpStream::connect(format!("{}:{}", address, port)) {
Ok(stream) => stream,
Err(err) => {
error!("Failed to connect: {}", err);
return Err(FileTransferError::new_ex(
FileTransferErrorType::ConnectionError,
err.to_string(),
));
}
fn connect(&mut self, params: &ProtocolParams) -> Result<Option<String>, FileTransferError> {
let params = match params.generic_params() {
Some(params) => params,
None => return Err(FileTransferError::new(FileTransferErrorType::BadAddress)),
};
// Get stream
info!("Connecting to {}:{}", params.address, params.port);
let mut stream: FtpStream =
match FtpStream::connect(format!("{}:{}", params.address, params.port)) {
Ok(stream) => stream,
Err(err) => {
error!("Failed to connect: {}", err);
return Err(FileTransferError::new_ex(
FileTransferErrorType::ConnectionError,
err.to_string(),
));
}
};
// If SSL, open secure session
if self.ftps {
info!("Setting up TLS stream...");
@@ -214,7 +213,7 @@ impl FileTransfer for FtpFileTransfer {
));
}
};
stream = match stream.into_secure(ctx, address.as_str()) {
stream = match stream.into_secure(ctx, params.address.as_str()) {
Ok(s) => s,
Err(err) => {
error!("Failed to setup TLS stream: {}", err);
@@ -226,12 +225,12 @@ impl FileTransfer for FtpFileTransfer {
};
}
// Login (use anonymous if credentials are unspecified)
let username: String = match username {
Some(u) => u,
let username: String = match &params.username {
Some(u) => u.to_string(),
None => String::from("anonymous"),
};
let password: String = match password {
Some(pwd) => pwd,
let password: String = match &params.password {
Some(pwd) => pwd.to_string(),
None => String::new(),
};
info!(
@@ -645,6 +644,7 @@ impl FileTransfer for FtpFileTransfer {
mod tests {
use super::*;
use crate::filetransfer::params::GenericProtocolParams;
use crate::utils::file::open_file;
#[cfg(feature = "with-containers")]
use crate::utils::test_helpers::write_file;
@@ -672,17 +672,15 @@ mod tests {
// Sample file
let (entry, file): (FsFile, tempfile::NamedTempFile) = create_sample_file_entry();
// Connect
#[cfg(not(feature = "github-actions"))]
let hostname: String = String::from("127.0.0.1");
#[cfg(feature = "github-actions")]
let hostname: String = String::from("127.0.0.1");
assert!(ftp
.connect(
hostname,
10021,
Some(String::from("test")),
Some(String::from("test")),
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address(hostname)
.port(10021)
.username(Some("test"))
.password(Some("test"))
))
.is_ok());
assert_eq!(ftp.is_connected(), true);
// Get pwd
@@ -810,12 +808,13 @@ mod tests {
let mut ftp: FtpFileTransfer = FtpFileTransfer::new(false);
// Connect
assert!(ftp
.connect(
String::from("127.0.0.1"),
10021,
Some(String::from("omar")),
Some(String::from("ommlar")),
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10021)
.username(Some("omar"))
.password(Some("ommlar"))
))
.is_err());
}
@@ -824,7 +823,13 @@ mod tests {
fn test_filetransfer_ftp_no_credentials() {
let mut ftp: FtpFileTransfer = FtpFileTransfer::new(false);
assert!(ftp
.connect(String::from("127.0.0.1"), 10021, None, None)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10021)
.username::<&str>(None)
.password::<&str>(None)
))
.is_err());
}
@@ -833,12 +838,13 @@ mod tests {
let mut ftp: FtpFileTransfer = FtpFileTransfer::new(false);
// Connect
assert!(ftp
.connect(
String::from("mybadserver.veryverybad.awful"),
21,
Some(String::from("omar")),
Some(String::from("ommlar")),
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("mybad.veribad.server")
.port(21)
.username::<&str>(None)
.password::<&str>(None)
))
.is_err());
}
@@ -890,12 +896,13 @@ mod tests {
let mut ftp: FtpFileTransfer = FtpFileTransfer::new(false);
// Connect
assert!(ftp
.connect(
String::from("test.rebex.net"),
21,
Some(String::from("demo")),
Some(String::from("password"))
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("test.rebex.net")
.port(21)
.username(Some("demo"))
.password(Some("password"))
))
.is_ok());
// Pwd
assert_eq!(ftp.pwd().ok().unwrap(), PathBuf::from("/"));

18
src/filetransfer/transfer/mod.rs Normal file
View File

@@ -0,0 +1,18 @@
//! # transfer
//!
//! This module exposes all the file transfers supported by termscp
// -- import
use super::{FileTransfer, FileTransferError, FileTransferErrorType, ProtocolParams};
// -- modules
mod ftp;
mod s3;
mod scp;
mod sftp;
// -- export
pub use self::s3::S3FileTransfer;
pub use ftp::FtpFileTransfer;
pub use scp::ScpFileTransfer;
pub use sftp::SftpFileTransfer;

697
src/filetransfer/transfer/s3/mod.rs Normal file
View File

@@ -0,0 +1,697 @@
//! ## S3 transfer
//!
//! S3 file transfer module
/**
* MIT License
*
* termscp - Copyright (c) 2021 Christian Visintin
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
// -- mod
mod object;
// Locals
use super::{FileTransfer, FileTransferError, FileTransferErrorType, ProtocolParams};
use crate::fs::{FsDirectory, FsEntry, FsFile};
use crate::utils::path;
use object::S3Object;
// ext
use s3::creds::Credentials;
use s3::serde_types::Object;
use s3::{Bucket, Region};
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::str::FromStr;
/// ## S3FileTransfer
///
/// Aws s3 file transfer
pub struct S3FileTransfer {
bucket: Option<Bucket>,
wrkdir: PathBuf,
}
impl Default for S3FileTransfer {
fn default() -> Self {
Self {
bucket: None,
wrkdir: PathBuf::from("/"),
}
}
}
impl S3FileTransfer {
/// ### list_objects
///
/// List objects contained in `p` path
fn list_objects(&self, p: &Path, list_dir: bool) -> Result<Vec<S3Object>, FileTransferError> {
// Make path relative
let key: String = Self::fmt_path(p, list_dir);
debug!("Query list directory {}; key: {}", p.display(), key);
self.query_objects(key, true)
}
/// ### stat_object
///
/// Stat an s3 object
fn stat_object(&self, p: &Path) -> Result<S3Object, FileTransferError> {
let key: String = Self::fmt_path(p, false);
debug!("Query stat object {}; key: {}", p.display(), key);
let objects = self.query_objects(key, false)?;
// Absolutize path
let absol: PathBuf = path::absolutize(Path::new("/"), p);
// Find associated object
match objects
.into_iter()
.find(|x| x.path.as_path() == absol.as_path())
{
Some(obj) => Ok(obj),
None => Err(FileTransferError::new_ex(
FileTransferErrorType::NoSuchFileOrDirectory,
format!("{}: No such file or directory", p.display()),
)),
}
}
/// ### query_objects
///
/// Query objects at key
fn query_objects(
&self,
key: String,
only_direct_children: bool,
) -> Result<Vec<S3Object>, FileTransferError> {
let results = self.bucket.as_ref().unwrap().list(key.clone(), None);
match results {
Ok(entries) => {
let mut objects: Vec<S3Object> = Vec::new();
entries.iter().for_each(|x| {
x.contents
.iter()
.filter(|x| {
if only_direct_children {
Self::list_object_should_be_kept(x, key.as_str())
} else {
true
}
})
.for_each(|x| objects.push(S3Object::from(x)))
});
debug!("Found objects: {:?}", objects);
Ok(objects)
}
Err(e) => Err(FileTransferError::new_ex(
FileTransferErrorType::DirStatFailed,
e.to_string(),
)),
}
}
/// ### list_object_should_be_kept
///
/// Returns whether object should be kept after list command.
/// The object won't be kept if:
///
/// 1. is not a direct child of provided dir
fn list_object_should_be_kept(obj: &Object, dir: &str) -> bool {
Self::is_direct_child(obj.key.as_str(), dir)
}
/// ### is_direct_child
///
/// Checks whether Object's key is direct child of `parent` path.
fn is_direct_child(key: &str, parent: &str) -> bool {
key == format!("{}{}", parent, S3Object::object_name(key))
|| key == format!("{}{}/", parent, S3Object::object_name(key))
}
/// ### resolve
///
/// Make s3 absolute path from a given path
fn resolve(&self, p: &Path) -> PathBuf {
path::diff_paths(path::absolutize(self.wrkdir.as_path(), p), &Path::new("/"))
.unwrap_or_default()
}
/// ### fmt_fs_entry_path
///
/// fmt path for fsentry according to format expected by s3
fn fmt_fs_file_path(f: &FsFile) -> String {
Self::fmt_path(f.abs_path.as_path(), false)
}
/// ### fmt_path
///
/// fmt path for fsentry according to format expected by s3
fn fmt_path(p: &Path, is_dir: bool) -> String {
// prevent root as slash
if p == Path::new("/") {
return "".to_string();
}
// Remove root only if absolute
#[cfg(target_family = "unix")]
let is_absolute: bool = p.is_absolute();
// NOTE: don't use is_absolute: on windows won't work
#[cfg(target_family = "windows")]
let is_absolute: bool = p.display().to_string().starts_with('/');
let p: PathBuf = match is_absolute {
true => path::diff_paths(p, &Path::new("/")).unwrap_or_default(),
false => p.to_path_buf(),
};
// NOTE: windows only: resolve paths
#[cfg(target_family = "windows")]
let p: PathBuf = PathBuf::from(path_slash::PathExt::to_slash_lossy(p.as_path()).as_str());
// Fmt
match is_dir {
true => {
let mut p: String = p.display().to_string();
if !p.ends_with('/') {
p.push('/');
}
p
}
false => p.to_string_lossy().to_string(),
}
}
}
impl FileTransfer for S3FileTransfer {
/// ### connect
///
/// Connect to the remote server
/// Can return banner / welcome message on success
fn connect(&mut self, params: &ProtocolParams) -> Result<Option<String>, FileTransferError> {
// Verify parameters are S3
let params = match params.s3_params() {
Some(params) => params,
None => return Err(FileTransferError::new(FileTransferErrorType::BadAddress)),
};
// Load credentials
debug!("Loading credentials... (profile {:?})", params.profile);
let credentials: Credentials =
Credentials::new(None, None, None, None, params.profile.as_deref()).map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::AuthenticationFailed,
format!("Could not load s3 credentials: {}", e),
)
})?;
// Parse region
debug!("Parsing region {}", params.region);
let region: Region = Region::from_str(params.region.as_str()).map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::AuthenticationFailed,
format!("Could not parse s3 region: {}", e),
)
})?;
debug!(
"Credentials loaded! Connecting to bucket {}...",
params.bucket_name
);
self.bucket = Some(
Bucket::new(params.bucket_name.as_str(), region, credentials).map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::AuthenticationFailed,
format!("Could not connect to bucket {}: {}", params.bucket_name, e),
)
})?,
);
info!("Connection successfully established");
Ok(None)
}
/// ### disconnect
///
/// Disconnect from the remote server
fn disconnect(&mut self) -> Result<(), FileTransferError> {
info!("Disconnecting from S3 bucket...");
match self.bucket.take() {
Some(bucket) => {
drop(bucket);
Ok(())
}
None => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### is_connected
///
/// Indicates whether the client is connected to remote
fn is_connected(&self) -> bool {
self.bucket.is_some()
}
/// ### pwd
///
/// Print working directory
fn pwd(&mut self) -> Result<PathBuf, FileTransferError> {
info!("PWD");
match self.is_connected() {
true => Ok(self.wrkdir.clone()),
false => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### change_dir
///
/// Change working directory
fn change_dir(&mut self, dir: &Path) -> Result<PathBuf, FileTransferError> {
match &self.bucket.is_some() {
true => {
// Always allow entering root
if dir == Path::new("/") {
self.wrkdir = dir.to_path_buf();
info!("New working directory: {}", self.wrkdir.display());
return Ok(self.wrkdir.clone());
}
// Check if directory exists
debug!("Entering directory {}...", dir.display());
let dir_p: PathBuf = self.resolve(dir);
let dir_s: String = Self::fmt_path(dir_p.as_path(), true);
debug!("Searching for key {} (path: {})...", dir_s, dir_p.display());
// Check if directory already exists
if self
.stat_object(PathBuf::from(dir_s.as_str()).as_path())
.is_ok()
{
self.wrkdir = path::absolutize(Path::new("/"), dir_p.as_path());
info!("New working directory: {}", self.wrkdir.display());
Ok(self.wrkdir.clone())
} else {
Err(FileTransferError::new(
FileTransferErrorType::NoSuchFileOrDirectory,
))
}
}
false => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### copy
///
/// Copy file to destination
fn copy(&mut self, _src: &FsEntry, _dst: &Path) -> Result<(), FileTransferError> {
Err(FileTransferError::new(
FileTransferErrorType::UnsupportedFeature,
))
}
/// ### list_dir
///
/// List directory entries
fn list_dir(&mut self, path: &Path) -> Result<Vec<FsEntry>, FileTransferError> {
match self.is_connected() {
true => self
.list_objects(path, true)
.map(|x| x.into_iter().map(|x| x.into()).collect()),
false => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### mkdir
///
/// Make directory
/// In case the directory already exists, it must return an Error of kind `FileTransferErrorType::DirectoryAlreadyExists`
fn mkdir(&mut self, dir: &Path) -> Result<(), FileTransferError> {
match &self.bucket {
Some(bucket) => {
let dir: String = Self::fmt_path(self.resolve(dir).as_path(), true);
debug!("Making directory {}...", dir);
// Check if directory already exists
if self
.stat_object(PathBuf::from(dir.as_str()).as_path())
.is_ok()
{
error!("Directory {} already exists", dir);
return Err(FileTransferError::new(
FileTransferErrorType::DirectoryAlreadyExists,
));
}
bucket
.put_object(dir.as_str(), &[])
.map(|_| ())
.map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::FileCreateDenied,
format!("Could not make directory: {}", e),
)
})
}
None => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### remove
///
/// Remove a file or a directory
fn remove(&mut self, file: &FsEntry) -> Result<(), FileTransferError> {
let path = Self::fmt_path(
path::diff_paths(file.get_abs_path(), &Path::new("/"))
.unwrap_or_default()
.as_path(),
file.is_dir(),
);
info!("Removing object {}...", path);
match &self.bucket {
Some(bucket) => bucket.delete_object(path).map(|_| ()).map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::ProtocolError,
format!("Could not remove file: {}", e),
)
}),
None => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### rename
///
/// Rename file or a directory
fn rename(&mut self, _file: &FsEntry, _dst: &Path) -> Result<(), FileTransferError> {
Err(FileTransferError::new(
FileTransferErrorType::UnsupportedFeature,
))
}
/// ### stat
///
/// Stat file and return FsEntry
fn stat(&mut self, p: &Path) -> Result<FsEntry, FileTransferError> {
match self.is_connected() {
true => {
// First try as a "file"
let path: PathBuf = self.resolve(p);
if let Ok(obj) = self.stat_object(path.as_path()) {
return Ok(obj.into());
}
// Try as a "directory"
debug!("Failed to stat object as file; trying as a directory...");
let path: PathBuf = PathBuf::from(Self::fmt_path(path.as_path(), true));
self.stat_object(path.as_path()).map(|x| x.into())
}
false => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### exec
///
/// Execute a command on remote host
fn exec(&mut self, _cmd: &str) -> Result<String, FileTransferError> {
Err(FileTransferError::new(
FileTransferErrorType::UnsupportedFeature,
))
}
/// ### send_file_wno_stream
///
/// Send a file to remote WITHOUT using streams.
/// This method SHOULD be implemented ONLY when streams are not supported by the current file transfer.
/// The developer implementing the filetransfer user should FIRST try with `send_file` followed by `on_sent`
/// If the function returns error kind() `UnsupportedFeature`, then he should call this function.
/// By default this function uses the streams function to copy content from reader to writer
fn send_file_wno_stream(
&mut self,
_src: &FsFile,
dest: &Path,
mut reader: Box<dyn Read>,
) -> Result<(), FileTransferError> {
match &mut self.bucket {
Some(bucket) => {
let key = Self::fmt_path(dest, false);
info!("Query PUT for key '{}'", key);
bucket
.put_object_stream(&mut reader, key.as_str())
.map(|_| ())
.map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::ProtocolError,
format!("Could not put file: {}", e),
)
})
}
None => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
/// ### recv_file_wno_stream
///
/// Receive a file from remote WITHOUT using streams.
/// This method SHOULD be implemented ONLY when streams are not supported by the current file transfer.
/// The developer implementing the filetransfer user should FIRST try with `send_file` followed by `on_sent`
/// If the function returns error kind() `UnsupportedFeature`, then he should call this function.
/// By default this function uses the streams function to copy content from reader to writer
fn recv_file_wno_stream(&mut self, src: &FsFile, dest: &Path) -> Result<(), FileTransferError> {
match &mut self.bucket {
Some(bucket) => {
let mut writer = File::create(dest).map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::FileCreateDenied,
format!("Could not open local file: {}", e),
)
})?;
let key = Self::fmt_fs_file_path(src);
info!("Query GET for key '{}'", key);
bucket
.get_object_stream(key.as_str(), &mut writer)
.map(|_| ())
.map_err(|e| {
FileTransferError::new_ex(
FileTransferErrorType::ProtocolError,
format!("Could not get file: {}", e),
)
})
}
None => Err(FileTransferError::new(
FileTransferErrorType::UninitializedSession,
)),
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[cfg(feature = "with-s3-ci")]
use crate::filetransfer::params::AwsS3Params;
#[cfg(feature = "with-s3-ci")]
use crate::utils::random;
use crate::utils::test_helpers;
use pretty_assertions::assert_eq;
#[cfg(feature = "with-s3-ci")]
use std::env;
#[cfg(feature = "with-s3-ci")]
use tempfile::NamedTempFile;
#[test]
fn s3_new() {
let s3: S3FileTransfer = S3FileTransfer::default();
assert_eq!(s3.wrkdir.as_path(), Path::new("/"));
assert!(s3.bucket.is_none());
}
#[test]
fn s3_is_direct_child() {
assert_eq!(S3FileTransfer::is_direct_child("pippo/", ""), true);
assert_eq!(
S3FileTransfer::is_direct_child("pippo/sottocartella/", ""),
false
);
assert_eq!(
S3FileTransfer::is_direct_child("pippo/sottocartella/", "pippo/"),
true
);
assert_eq!(
S3FileTransfer::is_direct_child("pippo/sottocartella/", "pippo"), // This case must be handled indeed
false
);
assert_eq!(
S3FileTransfer::is_direct_child(
"pippo/sottocartella/readme.md",
"pippo/sottocartella/"
),
true
);
assert_eq!(
S3FileTransfer::is_direct_child(
"pippo/sottocartella/readme.md",
"pippo/sottocartella/"
),
true
);
}
#[test]
fn s3_resolve() {
let mut s3: S3FileTransfer = S3FileTransfer::default();
s3.wrkdir = PathBuf::from("/tmp");
// Absolute
assert_eq!(
s3.resolve(&Path::new("/tmp/sottocartella/")).as_path(),
Path::new("tmp/sottocartella")
);
// Relative
assert_eq!(
s3.resolve(&Path::new("subfolder/")).as_path(),
Path::new("tmp/subfolder")
);
}
#[test]
fn s3_fmt_fs_file_path() {
let f: FsFile =
test_helpers::make_fsentry(&Path::new("/tmp/omar.txt"), false).unwrap_file();
assert_eq!(
S3FileTransfer::fmt_fs_file_path(&f).as_str(),
"tmp/omar.txt"
);
}
#[test]
fn s3_fmt_path() {
assert_eq!(
S3FileTransfer::fmt_path(&Path::new("/tmp/omar.txt"), false).as_str(),
"tmp/omar.txt"
);
assert_eq!(
S3FileTransfer::fmt_path(&Path::new("omar.txt"), false).as_str(),
"omar.txt"
);
assert_eq!(
S3FileTransfer::fmt_path(&Path::new("/tmp/subfolder"), true).as_str(),
"tmp/subfolder/"
);
assert_eq!(
S3FileTransfer::fmt_path(&Path::new("tmp/subfolder"), true).as_str(),
"tmp/subfolder/"
);
assert_eq!(
S3FileTransfer::fmt_path(&Path::new("tmp"), true).as_str(),
"tmp/"
);
assert_eq!(
S3FileTransfer::fmt_path(&Path::new("tmp/"), true).as_str(),
"tmp/"
);
assert_eq!(S3FileTransfer::fmt_path(&Path::new("/"), true).as_str(), "");
}
// -- test transfer
#[cfg(feature = "with-s3-ci")]
#[test]
fn s3_filetransfer() {
// Gather s3 environment args
let bucket: String = env::var("AWS_S3_BUCKET").ok().unwrap();
let region: String = env::var("AWS_S3_REGION").ok().unwrap();
let params = get_ftparams(bucket, region);
// Get transfer
let mut s3 = S3FileTransfer::default();
// Connect
assert!(s3.connect(&params).is_ok());
// Check is connected
assert_eq!(s3.is_connected(), true);
// Pwd
assert_eq!(s3.pwd().ok().unwrap(), PathBuf::from("/"));
// Go to github-ci directory
assert!(s3.change_dir(&Path::new("/github-ci")).is_ok());
assert_eq!(s3.pwd().ok().unwrap(), PathBuf::from("/github-ci"));
// Find
assert_eq!(s3.find("*.jpg").ok().unwrap().len(), 1);
// List directory (3 entries)
assert_eq!(s3.list_dir(&Path::new("/github-ci")).ok().unwrap().len(), 3);
// Go to playground
assert!(s3.change_dir(&Path::new("/github-ci/playground")).is_ok());
assert_eq!(
s3.pwd().ok().unwrap(),
PathBuf::from("/github-ci/playground")
);
// Create directory
let dir_name: String = format!("{}/", random::random_alphanumeric_with_len(8));
let mut dir_path: PathBuf = PathBuf::from("/github-ci/playground");
dir_path.push(dir_name.as_str());
let dir_entry = test_helpers::make_fsentry(dir_path.as_path(), true);
assert!(s3.mkdir(dir_path.as_path()).is_ok());
assert!(s3.change_dir(dir_path.as_path()).is_ok());
// Copy/rename file is unsupported
assert!(s3.copy(&dir_entry, &Path::new("/copia")).is_err());
assert!(s3.rename(&dir_entry, &Path::new("/copia")).is_err());
// Exec is unsupported
assert!(s3.exec("omar!").is_err());
// Stat file
let entry = s3
.stat(&Path::new("/github-ci/avril_lavigne.jpg"))
.ok()
.unwrap()
.unwrap_file();
assert_eq!(entry.name.as_str(), "avril_lavigne.jpg");
assert_eq!(
entry.abs_path.as_path(),
Path::new("/github-ci/avril_lavigne.jpg")
);
assert_eq!(entry.ftype.as_deref().unwrap(), "jpg");
assert_eq!(entry.size, 101738);
assert_eq!(entry.user, None);
assert_eq!(entry.group, None);
assert_eq!(entry.unix_pex, None);
// Download file
let (local_file_entry, local_file): (FsFile, NamedTempFile) =
test_helpers::create_sample_file_entry();
let remote_entry =
test_helpers::make_fsentry(&Path::new("/github-ci/avril_lavigne.jpg"), false)
.unwrap_file();
assert!(s3
.recv_file_wno_stream(&remote_entry, local_file.path())
.is_ok());
// Upload file
let mut dest_path = dir_path.clone();
dest_path.push("aurellia_lavagna.jpg");
let reader = Box::new(File::open(local_file.path()).ok().unwrap());
assert!(s3
.send_file_wno_stream(&local_file_entry, dest_path.as_path(), reader)
.is_ok());
// Remove temp dir
assert!(s3.remove(&dir_entry).is_ok());
// Disconnect
assert!(s3.disconnect().is_ok());
}
#[cfg(feature = "with-s3-ci")]
fn get_ftparams(bucket: String, region: String) -> ProtocolParams {
ProtocolParams::AwsS3(AwsS3Params::new(bucket, region, None))
}
}

247
src/filetransfer/transfer/s3/object.rs Normal file
View File

@@ -0,0 +1,247 @@
//! ## S3 object
//!
//! This module exposes the S3Object structure, which is an intermediate structure to work with
//! S3 objects. Easy to be converted into a FsEntry.
/**
* MIT License
*
* termscp - Copyright (c) 2021 Christian Visintin
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
use super::{FsDirectory, FsEntry, FsFile, Object};
use crate::utils::parser::parse_datetime;
use crate::utils::path;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};
/// ## S3Object
///
/// An intermediate struct to work with s3 `Object`.
/// Really easy to be converted into a `FsEntry`
#[derive(Debug)]
pub struct S3Object {
pub name: String,
pub path: PathBuf,
pub size: usize,
pub last_modified: SystemTime,
/// Whether or not represents a directory. I already know directories don't exist in s3!
pub is_dir: bool,
}
impl From<&Object> for S3Object {
fn from(obj: &Object) -> Self {
let is_dir: bool = obj.key.ends_with('/');
let abs_path: PathBuf = path::absolutize(
PathBuf::from("/").as_path(),
PathBuf::from(obj.key.as_str()).as_path(),
);
let last_modified: SystemTime =
match parse_datetime(obj.last_modified.as_str(), "%Y-%m-%dT%H:%M:%S%Z") {
Ok(dt) => dt,
Err(_) => UNIX_EPOCH,
};
Self {
name: Self::object_name(obj.key.as_str()),
path: abs_path,
size: obj.size as usize,
last_modified,
is_dir,
}
}
}
impl From<S3Object> for FsEntry {
fn from(obj: S3Object) -> Self {
let abs_path: PathBuf = path::absolutize(Path::new("/"), obj.path.as_path());
match obj.is_dir {
true => FsEntry::Directory(FsDirectory {
name: obj.name,
abs_path,
last_change_time: obj.last_modified,
last_access_time: obj.last_modified,
creation_time: obj.last_modified,
symlink: None,
user: None,
group: None,
unix_pex: None,
}),
false => FsEntry::File(FsFile {
name: obj.name,
ftype: obj
.path
.extension()
.map(|x| x.to_string_lossy().to_string()),
abs_path,
size: obj.size,
last_change_time: obj.last_modified,
last_access_time: obj.last_modified,
creation_time: obj.last_modified,
symlink: None,
user: None,
group: None,
unix_pex: None,
}),
}
}
}
impl S3Object {
/// ### object_name
///
/// Get object name from key
pub fn object_name(key: &str) -> String {
let mut tokens = key.split('/');
let count = tokens.clone().count();
let demi_last: String = match count > 1 {
true => tokens.nth(count - 2).unwrap().to_string(),
false => String::new(),
};
if let Some(last) = tokens.last() {
// If last is not empty, return last one
if !last.is_empty() {
return last.to_string();
}
}
// Return demi last
demi_last
}
}
#[cfg(test)]
mod test {
use super::*;
use pretty_assertions::assert_eq;
use std::time::Duration;
#[test]
fn object_to_s3object_file() {
let obj: Object = Object {
key: String::from("pippo/sottocartella/chiedo.gif"),
e_tag: String::default(),
size: 1516966,
owner: None,
storage_class: String::default(),
last_modified: String::from("2021-08-28T10:20:37.000Z"),
};
let s3_obj: S3Object = S3Object::from(&obj);
assert_eq!(s3_obj.name.as_str(), "chiedo.gif");
assert_eq!(
s3_obj.path.as_path(),
Path::new("/pippo/sottocartella/chiedo.gif")
);
assert_eq!(s3_obj.size, 1516966);
assert_eq!(s3_obj.is_dir, false);
assert_eq!(
s3_obj
.last_modified
.duration_since(SystemTime::UNIX_EPOCH)
.ok()
.unwrap(),
Duration::from_secs(1630146037)
);
}
#[test]
fn object_to_s3object_dir() {
let obj: Object = Object {
key: String::from("temp/"),
e_tag: String::default(),
size: 0,
owner: None,
storage_class: String::default(),
last_modified: String::from("2021-08-28T10:20:37.000Z"),
};
let s3_obj: S3Object = S3Object::from(&obj);
assert_eq!(s3_obj.name.as_str(), "temp");
assert_eq!(s3_obj.path.as_path(), Path::new("/temp"));
assert_eq!(s3_obj.size, 0);
assert_eq!(s3_obj.is_dir, true);
assert_eq!(
s3_obj
.last_modified
.duration_since(SystemTime::UNIX_EPOCH)
.ok()
.unwrap(),
Duration::from_secs(1630146037)
);
}
#[test]
fn fsentry_from_s3obj_file() {
let obj: S3Object = S3Object {
name: String::from("chiedo.gif"),
path: PathBuf::from("/pippo/sottocartella/chiedo.gif"),
size: 1516966,
is_dir: false,
last_modified: UNIX_EPOCH,
};
let entry: FsFile = FsEntry::from(obj).unwrap_file();
assert_eq!(entry.name.as_str(), "chiedo.gif");
assert_eq!(
entry.abs_path.as_path(),
Path::new("/pippo/sottocartella/chiedo.gif")
);
assert_eq!(entry.creation_time, UNIX_EPOCH);
assert_eq!(entry.last_change_time, UNIX_EPOCH);
assert_eq!(entry.last_access_time, UNIX_EPOCH);
assert_eq!(entry.size, 1516966);
assert_eq!(entry.ftype.unwrap().as_str(), "gif");
assert_eq!(entry.user, None);
assert_eq!(entry.group, None);
assert_eq!(entry.unix_pex, None);
}
#[test]
fn fsentry_from_s3obj_directory() {
let obj: S3Object = S3Object {
name: String::from("temp"),
path: PathBuf::from("/temp"),
size: 0,
is_dir: true,
last_modified: UNIX_EPOCH,
};
let entry: FsDirectory = FsEntry::from(obj).unwrap_dir();
assert_eq!(entry.name.as_str(), "temp");
assert_eq!(entry.abs_path.as_path(), Path::new("/temp"));
assert_eq!(entry.creation_time, UNIX_EPOCH);
assert_eq!(entry.last_change_time, UNIX_EPOCH);
assert_eq!(entry.last_access_time, UNIX_EPOCH);
assert_eq!(entry.user, None);
assert_eq!(entry.group, None);
assert_eq!(entry.unix_pex, None);
}
#[test]
fn object_name() {
assert_eq!(
S3Object::object_name("pippo/sottocartella/chiedo.gif").as_str(),
"chiedo.gif"
);
assert_eq!(
S3Object::object_name("pippo/sottocartella/").as_str(),
"sottocartella"
);
assert_eq!(S3Object::object_name("pippo/").as_str(), "pippo");
}
}

122
src/filetransfer/scp_transfer.rs → src/filetransfer/transfer/scp.rs
View File

@@ -1,4 +1,4 @@
//! ## SCP_Transfer
//! ## SCP transfer
//!
//! `scps_transfer` is the module which provides the implementation for the SCP file transfer
@@ -26,7 +26,7 @@
* SOFTWARE.
*/
// Locals
use super::{FileTransfer, FileTransferError, FileTransferErrorType};
use super::{FileTransfer, FileTransferError, FileTransferErrorType, ProtocolParams};
use crate::fs::{FsDirectory, FsEntry, FsFile, UnixPex};
use crate::system::sshkey_storage::SshKeyStorage;
use crate::utils::fmt::{fmt_time, shadow_password};
@@ -333,17 +333,15 @@ impl FileTransfer for ScpFileTransfer {
/// ### connect
///
/// Connect to the remote server
fn connect(
&mut self,
address: String,
port: u16,
username: Option<String>,
password: Option<String>,
) -> Result<Option<String>, FileTransferError> {
fn connect(&mut self, params: &ProtocolParams) -> Result<Option<String>, FileTransferError> {
let params = match params.generic_params() {
Some(params) => params,
None => return Err(FileTransferError::new(FileTransferErrorType::BadAddress)),
};
// Setup tcp stream
info!("Connecting to {}:{}", address, port);
info!("Connecting to {}:{}", params.address, params.port);
let socket_addresses: Vec<SocketAddr> =
match format!("{}:{}", address, port).to_socket_addrs() {
match format!("{}:{}", params.address, params.port).to_socket_addrs() {
Ok(s) => s.collect(),
Err(err) => {
return Err(FileTransferError::new_ex(
@@ -398,14 +396,14 @@ impl FileTransfer for ScpFileTransfer {
err.to_string(),
));
}
let username: String = match username {
Some(u) => u,
let username: String = match &params.username {
Some(u) => u.to_string(),
None => String::from(""),
};
// Check if it is possible to authenticate using a RSA key
match self
.key_storage
.resolve(address.as_str(), username.as_str())
.resolve(params.address.as_str(), username.as_str())
{
Some(rsa_key) => {
debug!(
@@ -418,7 +416,7 @@ impl FileTransfer for ScpFileTransfer {
username.as_str(),
None,
rsa_key.as_path(),
password.as_deref(),
params.password.as_deref(),
) {
error!("Authentication failed: {}", err);
return Err(FileTransferError::new_ex(
@@ -432,11 +430,16 @@ impl FileTransfer for ScpFileTransfer {
debug!(
"Authenticating with username {} and password {}",
username,
shadow_password(password.as_deref().unwrap_or(""))
shadow_password(params.password.as_deref().unwrap_or(""))
);
if let Err(err) = session.userauth_password(
username.as_str(),
password.unwrap_or_else(|| String::from("")).as_str(),
params
.password
.as_ref()
.cloned()
.unwrap_or_else(|| String::from(""))
.as_str(),
) {
error!("Authentication failed: {}", err);
return Err(FileTransferError::new_ex(
@@ -942,36 +945,13 @@ impl FileTransfer for ScpFileTransfer {
)),
}
}
/// ### on_sent
///
/// Finalize send method.
/// This method must be implemented only if necessary; in case you don't need it, just return `Ok(())`
/// The purpose of this method is to finalize the connection with the peer when writing data.
/// This is necessary for some protocols such as FTP.
/// You must call this method each time you want to finalize the write of the remote file.
fn on_sent(&mut self, _writable: Box<dyn Write>) -> Result<(), FileTransferError> {
// Nothing to do
Ok(())
}
/// ### on_recv
///
/// Finalize recv method.
/// This method must be implemented only if necessary; in case you don't need it, just return `Ok(())`
/// The purpose of this method is to finalize the connection with the peer when reading data.
/// This mighe be necessary for some protocols.
/// You must call this method each time you want to finalize the read of the remote file.
fn on_recv(&mut self, _readable: Box<dyn Read>) -> Result<(), FileTransferError> {
// Nothing to do
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::filetransfer::params::GenericProtocolParams;
use crate::utils::test_helpers::make_fsentry;
use pretty_assertions::assert_eq;
@@ -993,12 +973,13 @@ mod tests {
let (entry, file): (FsFile, tempfile::NamedTempFile) = create_sample_file_entry();
// Connect
assert!(client
.connect(
String::from("127.0.0.1"),
10222,
Some(String::from("sftp")),
Some(String::from("password"))
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10222)
.username(Some("sftp"))
.password(Some("password"))
))
.is_ok());
// Check session and sftp
assert!(client.session.is_some());
@@ -1180,12 +1161,13 @@ mod tests {
let mut client: ScpFileTransfer = ScpFileTransfer::new(storage);
// Connect
assert!(client
.connect(
String::from("127.0.0.1"),
10222,
Some(String::from("sftp")),
None,
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10222)
.username(Some("sftp"))
.password::<&str>(None)
))
.is_ok());
assert_eq!(client.is_connected(), true);
assert!(client.disconnect().is_ok());
@@ -1195,12 +1177,13 @@ mod tests {
fn test_filetransfer_scp_bad_auth() {
let mut client: ScpFileTransfer = ScpFileTransfer::new(SshKeyStorage::empty());
assert!(client
.connect(
String::from("127.0.0.1"),
10222,
Some(String::from("demo")),
Some(String::from("badpassword"))
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10222)
.username(Some("sftp"))
.password(Some("badpassword"))
))
.is_err());
}
@@ -1209,7 +1192,13 @@ mod tests {
fn test_filetransfer_scp_no_credentials() {
let mut client: ScpFileTransfer = ScpFileTransfer::new(SshKeyStorage::empty());
assert!(client
.connect(String::from("127.0.0.1"), 10222, None, None)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10222)
.username::<&str>(None)
.password::<&str>(None)
))
.is_err());
}
@@ -1217,12 +1206,13 @@ mod tests {
fn test_filetransfer_scp_bad_server() {
let mut client: ScpFileTransfer = ScpFileTransfer::new(SshKeyStorage::empty());
assert!(client
.connect(
String::from("mybadserver.veryverybad.awful"),
22,
None,
None
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("mybad.verybad.server")
.port(10222)
.username(Some("sftp"))
.password(Some("password"))
))
.is_err());
}

149
src/filetransfer/sftp_transfer.rs → src/filetransfer/transfer/sftp.rs
View File

@@ -1,4 +1,4 @@
//! ## SFTP_Transfer
//! ## SFTP transfer
//!
//! `sftp_transfer` is the module which provides the implementation for the SFTP file transfer
@@ -26,7 +26,7 @@
* SOFTWARE.
*/
// Locals
use super::{FileTransfer, FileTransferError, FileTransferErrorType};
use super::{FileTransfer, FileTransferError, FileTransferErrorType, ProtocolParams};
use crate::fs::{FsDirectory, FsEntry, FsFile, UnixPex};
use crate::system::sshkey_storage::SshKeyStorage;
use crate::utils::fmt::{fmt_time, shadow_password};
@@ -257,17 +257,15 @@ impl FileTransfer for SftpFileTransfer {
/// ### connect
///
/// Connect to the remote server
fn connect(
&mut self,
address: String,
port: u16,
username: Option<String>,
password: Option<String>,
) -> Result<Option<String>, FileTransferError> {
fn connect(&mut self, params: &ProtocolParams) -> Result<Option<String>, FileTransferError> {
let params = match params.generic_params() {
Some(params) => params,
None => return Err(FileTransferError::new(FileTransferErrorType::BadAddress)),
};
// Setup tcp stream
info!("Connecting to {}:{}", address, port);
info!("Connecting to {}:{}", params.address, params.port);
let socket_addresses: Vec<SocketAddr> =
match format!("{}:{}", address, port).to_socket_addrs() {
match format!("{}:{}", params.address, params.port).to_socket_addrs() {
Ok(s) => s.collect(),
Err(err) => {
return Err(FileTransferError::new_ex(
@@ -321,14 +319,14 @@ impl FileTransfer for SftpFileTransfer {
err.to_string(),
));
}
let username: String = match username {
Some(u) => u,
let username: String = match &params.username {
Some(u) => u.to_string(),
None => String::from(""),
};
// Check if it is possible to authenticate using a RSA key
match self
.key_storage
.resolve(address.as_str(), username.as_str())
.resolve(params.address.as_str(), username.as_str())
{
Some(rsa_key) => {
debug!(
@@ -341,7 +339,7 @@ impl FileTransfer for SftpFileTransfer {
username.as_str(),
None,
rsa_key.as_path(),
password.as_deref(),
params.password.as_deref(),
) {
error!("Authentication failed: {}", err);
return Err(FileTransferError::new_ex(
@@ -355,11 +353,16 @@ impl FileTransfer for SftpFileTransfer {
debug!(
"Authenticating with username {} and password {}",
username,
shadow_password(password.as_deref().unwrap_or(""))
shadow_password(params.password.as_deref().unwrap_or(""))
);
if let Err(err) = session.userauth_password(
username.as_str(),
password.unwrap_or_else(|| String::from("")).as_str(),
params
.password
.as_ref()
.cloned()
.unwrap_or_else(|| String::from(""))
.as_str(),
) {
error!("Authentication failed: {}", err);
return Err(FileTransferError::new_ex(
@@ -766,36 +769,21 @@ impl FileTransfer for SftpFileTransfer {
}
}
}
/// ### on_sent
///
/// Finalize send method. This method must be implemented only if necessary.
/// The purpose of this method is to finalize the connection with the peer when writing data.
/// This is necessary for some protocols such as FTP.
/// You must call this method each time you want to finalize the write of the remote file.
fn on_sent(&mut self, _writable: Box<dyn Write>) -> Result<(), FileTransferError> {
Ok(())
}
/// ### on_recv
///
/// Finalize recv method. This method must be implemented only if necessary.
/// The purpose of this method is to finalize the connection with the peer when reading data.
/// This mighe be necessary for some protocols.
/// You must call this method each time you want to finalize the read of the remote file.
fn on_recv(&mut self, _readable: Box<dyn Read>) -> Result<(), FileTransferError> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::filetransfer::params::GenericProtocolParams;
use crate::utils::test_helpers::make_fsentry;
#[cfg(feature = "with-containers")]
use crate::utils::test_helpers::{create_sample_file_entry, write_file, write_ssh_key};
use crate::utils::test_helpers::{
create_sample_file, create_sample_file_entry, write_file, write_ssh_key,
};
use pretty_assertions::assert_eq;
#[cfg(feature = "with-containers")]
use std::fs::File;
#[test]
fn test_filetransfer_sftp_new() {
@@ -814,12 +802,13 @@ mod tests {
let (entry, file): (FsFile, tempfile::NamedTempFile) = create_sample_file_entry();
// Connect
assert!(client
.connect(
String::from("127.0.0.1"),
10022,
Some(String::from("sftp")),
Some(String::from("password"))
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10022)
.username(Some("sftp"))
.password(Some("password"))
))
.is_ok());
// Check session and sftp
assert!(client.session.is_some());
@@ -889,6 +878,11 @@ mod tests {
.unwrap();
write_file(&file, &mut writable);
assert!(client.on_sent(writable).is_ok());
// Upload file without stream
let reader = Box::new(File::open(entry.abs_path.as_path()).ok().unwrap());
assert!(client
.send_file_wno_stream(&entry, PathBuf::from("README2.md").as_path(), reader)
.is_ok());
// Upload file (err)
assert!(client
.send_file(&entry, PathBuf::from("/ommlar/omarone").as_path())
@@ -898,10 +892,10 @@ mod tests {
.list_dir(PathBuf::from("/tmp/omar").as_path())
.ok()
.unwrap();
assert_eq!(list.len(), 2);
assert_eq!(list.len(), 3);
// Find
assert_eq!(client.find("*.txt").ok().unwrap().len(), 1);
assert_eq!(client.find("*.md").ok().unwrap().len(), 1);
assert_eq!(client.find("*.md").ok().unwrap().len(), 2);
assert_eq!(client.find("*.jpeg").ok().unwrap().len(), 0);
// Rename
assert!(client
@@ -955,6 +949,9 @@ mod tests {
let mut data: Vec<u8> = vec![0; 1024];
assert!(readable.read(&mut data).is_ok());
assert!(client.on_recv(readable).is_ok());
let dest_file = create_sample_file();
// Receive file wno stream
assert!(client.recv_file_wno_stream(&file, dest_file.path()).is_ok());
// Receive file (err)
assert!(client.recv_file(&entry).is_err());
// Cleanup
@@ -979,12 +976,13 @@ mod tests {
let mut client: SftpFileTransfer = SftpFileTransfer::new(storage);
// Connect
assert!(client
.connect(
String::from("127.0.0.1"),
10022,
Some(String::from("sftp")),
None,
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10022)
.username(Some("sftp"))
.password::<&str>(None)
))
.is_ok());
assert_eq!(client.is_connected(), true);
assert!(client.disconnect().is_ok());
@@ -994,12 +992,13 @@ mod tests {
fn test_filetransfer_sftp_bad_auth() {
let mut client: SftpFileTransfer = SftpFileTransfer::new(SshKeyStorage::empty());
assert!(client
.connect(
String::from("127.0.0.1"),
10022,
Some(String::from("demo")),
Some(String::from("badpassword"))
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10022)
.username(Some("sftp"))
.password(Some("badpassword"))
))
.is_err());
}
@@ -1008,7 +1007,13 @@ mod tests {
fn test_filetransfer_sftp_no_credentials() {
let mut client: SftpFileTransfer = SftpFileTransfer::new(SshKeyStorage::empty());
assert!(client
.connect(String::from("127.0.0.1"), 10022, None, None)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10022)
.username::<&str>(None)
.password::<&str>(None)
))
.is_err());
}
@@ -1018,12 +1023,13 @@ mod tests {
let mut client: SftpFileTransfer = SftpFileTransfer::new(SshKeyStorage::empty());
// Connect
assert!(client
.connect(
String::from("127.0.0.1"),
10022,
Some(String::from("sftp")),
Some(String::from("password"))
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("127.0.0.1")
.port(10022)
.username(Some("sftp"))
.password(Some("password"))
))
.is_ok());
// get realpath
assert!(client
@@ -1054,12 +1060,13 @@ mod tests {
fn test_filetransfer_sftp_bad_server() {
let mut client: SftpFileTransfer = SftpFileTransfer::new(SshKeyStorage::empty());
assert!(client
.connect(
String::from("mybadserver.veryverybad.awful"),
22,
None,
None
)
.connect(&ProtocolParams::Generic(
GenericProtocolParams::default()
.address("myverybad.verybad.server")
.port(10022)
.username(Some("sftp"))
.password(Some("password"))
))
.is_err());
}