Optimize transfers (#147)

* When the file is exchanged, all times attributes are set (if supported by the protocol)

* If local/remote file have the same last modification time (`mtime`), the file is not transferred
This commit is contained in:
Christian Visintin
2023-02-12 22:27:42 +01:00
committed by GitHub
parent 54aecdc478
commit 82330f870e
5 changed files with 149 additions and 42 deletions

View File

@@ -9,7 +9,7 @@ use crate::utils::fmt::fmt_millis;
// Ext
use bytesize::ByteSize;
use remotefs::fs::{File, ReadStream, UnixPex, Welcome, WriteStream};
use remotefs::fs::{File, Metadata, ReadStream, UnixPex, Welcome, WriteStream};
use remotefs::{RemoteError, RemoteErrorType};
use std::fs::File as StdFile;
use std::io::{Read, Seek, Write};
@@ -405,6 +405,18 @@ impl FileTransferActivity {
.stat(local.path.as_path())
.map_err(TransferErrorReason::HostError)
.map(|x| x.metadata().clone())?;
if !self.has_remote_file_changed(remote, &metadata) {
self.log(
LogLevel::Info,
format!(
"file {} won't be transferred since hasn't changed",
local.path().display()
),
);
self.transfer.full.update_progress(metadata.size as usize);
return Ok(());
}
// Upload file
// Try to open local file
match self.host.open_file_read(local.path.as_path()) {
@@ -507,6 +519,10 @@ impl FileTransferActivity {
if self.transfer.aborted() {
return Err(TransferErrorReason::Abrupted);
}
// set stat
if let Err(err) = self.client.setstat(remote, local.metadata().clone()) {
error!("failed to set stat for {}: {}", remote.display(), err);
}
self.log(
LogLevel::Info,
format!(
@@ -549,6 +565,10 @@ impl FileTransferActivity {
if let Err(err) = self.client.create_file(remote, &metadata, Box::new(reader)) {
return Err(TransferErrorReason::FileTransferError(err));
}
// set stat
if let Err(err) = self.client.setstat(remote, metadata) {
error!("failed to set stat for {}: {}", remote.display(), err);
}
// Set transfer size ok
self.transfer.partial.update_progress(file_size);
self.transfer.full.update_progress(file_size);
@@ -684,19 +704,19 @@ impl FileTransferActivity {
match self.host.mkdir_ex(local_dir_path.as_path(), true) {
Ok(_) => {
// Apply file mode to directory
#[cfg(any(target_family = "unix", target_os = "macos", target_os = "linux"))]
if let Some(mode) = entry.metadata().mode {
if let Err(err) = self.host.chmod(local_dir_path.as_path(), mode) {
self.log(
LogLevel::Error,
format!(
"Could not apply file mode {:o} to \"{}\": {}",
u32::from(mode),
local_dir_path.display(),
err
),
);
}
if let Err(err) = self
.host
.setstat(local_dir_path.as_path(), entry.metadata())
{
self.log(
LogLevel::Error,
format!(
"Could not set stat to directory {:?} to \"{}\": {}",
entry.metadata(),
local_dir_path.display(),
err
),
);
}
self.log(
LogLevel::Info,
@@ -812,6 +832,21 @@ impl FileTransferActivity {
remote: &File,
file_name: String,
) -> Result<(), TransferErrorReason> {
// check if files are equal (in case, don't transfer)
if !self.has_local_file_changed(local, remote) {
self.log(
LogLevel::Info,
format!(
"file {} won't be transferred since hasn't changed",
remote.path().display()
),
);
self.transfer
.full
.update_progress(remote.metadata().size as usize);
return Ok(());
}
// Try to open local file
match self.host.open_file_write(local) {
Ok(local_file) => {
@@ -909,19 +944,16 @@ impl FileTransferActivity {
return Err(TransferErrorReason::Abrupted);
}
// Apply file mode to file
#[cfg(target_family = "unix")]
if let Some(mode) = remote.metadata.mode {
if let Err(err) = self.host.chmod(local, mode) {
self.log(
LogLevel::Error,
format!(
"Could not apply file mode {:o} to \"{}\": {}",
u32::from(mode),
local.display(),
err
),
);
}
if let Err(err) = self.host.setstat(local, remote.metadata()) {
self.log(
LogLevel::Error,
format!(
"Could not set stat to file {:?} to \"{}\": {}",
remote.metadata(),
local.display(),
err
),
);
}
// Log
self.log(
@@ -970,19 +1002,16 @@ impl FileTransferActivity {
self.update_progress_bar(format!("Downloading \"{file_name}\""));
self.view();
// Apply file mode to file
#[cfg(target_family = "unix")]
if let Some(mode) = remote.metadata.mode {
if let Err(err) = self.host.chmod(local, mode) {
self.log(
LogLevel::Error,
format!(
"Could not apply file mode {:o} to \"{}\": {}",
u32::from(mode),
local.display(),
err
),
);
}
if let Err(err) = self.host.setstat(local, remote.metadata()) {
self.log(
LogLevel::Error,
format!(
"Could not set stat to file {:?} to \"{}\": {}",
remote.metadata(),
local.display(),
err
),
);
}
// Log
self.log(
@@ -1134,6 +1163,30 @@ impl FileTransferActivity {
}
}
// file changed
/// Check whether provided file has changed on local disk, compared to remote file
fn has_local_file_changed(&self, local: &Path, remote: &File) -> bool {
// check if files are equal (in case, don't transfer)
if let Ok(local_file) = self.host.stat(local) {
local_file.metadata().modified != remote.metadata().modified
|| local_file.metadata().size != remote.metadata().size
} else {
true
}
}
/// Checks whether remote file has changed compared to local file
fn has_remote_file_changed(&mut self, remote: &Path, local_metadata: &Metadata) -> bool {
// check if files are equal (in case, don't transfer)
if let Ok(remote_file) = self.client.stat(remote) {
local_metadata.modified != remote_file.metadata().modified
|| local_metadata.size != remote_file.metadata().size
} else {
true
}
}
// -- file exist
pub(crate) fn local_file_exists(&mut self, p: &Path) -> bool {