From 673f641fc9839345b4a761a8c548a40c7f22f96d Mon Sep 17 00:00:00 2001
From: goymann <philipp.goymann@mpi-bn.mpg.de>
Date: Sun, 4 Jun 2023 19:04:02 +0000
Subject: [PATCH] update
---
loosolab_s3/cmd.py | 13 +++++++-----
loosolab_s3/s3_functions.py | 42 ++++++++++++++++---------------------
2 files changed, 26 insertions(+), 29 deletions(-)
diff --git a/loosolab_s3/cmd.py b/loosolab_s3/cmd.py
index 3a9089e..b168e3d 100644
--- a/loosolab_s3/cmd.py
+++ b/loosolab_s3/cmd.py
@@ -27,11 +27,13 @@ def run_s3_functions():
s3 = Loosolab_s3(credentials, logger=False)
else:
s3 = Loosolab_s3(credentials)
+
+ if args.output:
+ output = args.output
+ else:
+ output = os.getcwd()
if args.read_form_dict:
-
- bucket_dict = args.read_form_dict
-
-
+ bucket_dict = args.read_form_dict
bucketname = bucket_dict.split(':')[0]
files = [bucket_dict.split(':')[1]]
if args.bucketname:
@@ -61,7 +63,7 @@ def run_s3_functions():
else:
utils.check_argparser(args, ['secret', "key", "files", "bucketname"])
- s3.download_s3_objects( bucketname, files, destination=os.getcwd(), download_bar=True)
+ s3.download_s3_objects( bucketname, files, destination=output, download_bar=True)
#--------------------------------------------------------------------------------------------------------#
# parse command line arguments:
def argparsefunc():
@@ -83,6 +85,7 @@ def argparsefunc():
parser.add_argument("--files", nargs='+', help="bucket upload files")
parser.add_argument("--bucketname")
parser.add_argument("--no_log", action='store_true', help='disable logging')
+ parser.add_argument("--output", help='Download folder')
parser.add_argument('-h', '--help', action='store_true', dest='show_help')
args = parser.parse_args()
diff --git a/loosolab_s3/s3_functions.py b/loosolab_s3/s3_functions.py
index a5b5332..7ea713b 100644
--- a/loosolab_s3/s3_functions.py
+++ b/loosolab_s3/s3_functions.py
@@ -215,7 +215,7 @@ class Loosolab_s3:
local_tag = self.etag_checksum(local_file_name)
else:
local_tag = hashlib.md5(open(local_file_name).read().encode('utf-8')).hexdigest()
- print('\n')
+
self.logger.info('local e-tag of ' + local_file_name +' is : ' + local_tag)
self.logger.info('s3 e-tag of ' + file_name +' is : ' + s3_e_tag)
@@ -475,25 +475,11 @@ class Loosolab_s3:
timeout : float
how long to watch for file in minutes
"""
+
+
bucket = self.session.Bucket(bucket_name)
modBool = False
# check if exists
- for local_file in file_list:
- file_name = os.path.basename(local_file)
- file_path = os.path.join(destination, local_file)
- if compare:
- modBool = self.compare_s3_etag(bucket_name, file_name, file_path)
- if self.check_s3_object_ex(bucket_name, file_name) and not modBool:
- if not os.path.exists(os.path.dirname(file_path)):
- os.makedirs(os.path.dirname(file_path))
- self.logger.info('Created directory: ' + os.path.dirname(file_path))
- if self.multipart_upload and not download_bar:
- bucket.download_file(file_name, file_path)
- elif download_bar:
-
- self.download(file_path,bucket_name, file_name)
- else:
- bucket.download_file(file_name, file_path, Config=self.transfer)
try:
for local_file in file_list:
file_name = os.path.basename(local_file)
@@ -501,11 +487,14 @@ class Loosolab_s3:
if compare:
modBool = self.compare_s3_etag(bucket_name, file_name, file_path)
if self.check_s3_object_ex(bucket_name, file_name) and not modBool:
- if not os.path.exists(file_path):
+ if not os.path.exists(os.path.dirname(file_path)):
os.makedirs(os.path.dirname(file_path))
self.logger.info('Created directory: ' + os.path.dirname(file_path))
- if self.multipart_upload:
+ if self.multipart_upload and not download_bar:
bucket.download_file(file_name, file_path)
+ elif download_bar:
+
+ self.download(file_path,bucket_name, file_name)
else:
bucket.download_file(file_name, file_path, Config=self.transfer)
except Exception as e:
@@ -514,18 +503,23 @@ class Loosolab_s3:
#--------------------------------------------------------------------------------------------------------#
def download(self, local_file_name, s3_bucket, s3_object_key):
-
+
meta_data = self.client.head_object(Bucket=s3_bucket, Key=s3_object_key)
- total_length = int(meta_data.get('ContentLength', 0))
+ total_length = float(meta_data.get('ContentLength', 0))
downloaded = 0
-
+ part = total_length / 10
+ count = total_length / 10
def progress(chunk):
nonlocal downloaded
+ nonlocal part
+ nonlocal count
downloaded += chunk
- done = int(50 * downloaded / total_length)
+ if downloaded > count:
+ print(str(int((100 / total_length ) * count)) + ' % Downloaded!')
+ count += part
- print("\r[%s%s]" % ('=' * done, ' ' * (50-done)) , end="\r")
+ #print("\r[%s%s]" % ('=' * done, ' ' * (50-done)) , end="\r")
#sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) )
#sys.stdout.flush()
--
GitLab