Skip to content
Snippets Groups Projects
Commit b6d8c1be authored by marina.kiweler01's avatar marina.kiweler01
Browse files

pattern included in get functions

parent 04994e09
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 78 additions and 113 deletions
s3_functions.py
+ 78
− 113
View file @ b6d8c1be
......@@ -14,6 +14,7 @@ import hashlib
import logging
import os
import random
import re
import sys
import time
......@@ -69,7 +70,7 @@ class Loosolab_s3:
)
self.session = session
self.check_s3_credentials() # list-bucket not allowed -> skip
self.check_s3_credentials()
#--------------------------------------------------------------------------------------------------------#
def create_s3_transfer(self, credentials):
......@@ -213,6 +214,7 @@ class Loosolab_s3:
--------
bucket_name : string
"""
bucket_name = bucket_name.lower() #only lowercase is allowed
if ('_' or '.') in bucket_name: #bucket must not contain '_' or '.'
bucket_name = bucket_name.replace('_','-').replace('.','')
self.my_logger.warning('There are not supported characters in your bucket name (\".\" or \"_\") they are replaced. New name is: ' + bucket_name)
......@@ -261,8 +263,12 @@ class Loosolab_s3:
#--------------------------------------------------------------------------------------------------------#
#------------- Bucket Management ------------------------------------------------------------------------#
def get_bucket_names(self):
def get_bucket_names(self, pattern=""):
""" Get a list of all buckets
Parameter:
----------
pattern : string
Returns:
--------
bucket_name_list : list
......@@ -272,31 +278,19 @@ class Loosolab_s3:
try:
for bucket in self.session.buckets.all():
bucket_name_list.append(bucket.name)
return bucket_name_list
except Exception as e:
self.my_logger.error('Buckets could not be listed. ' + str(e))
if not pattern: # retun ALL bucket names
return bucket_name_list
#--------------------------------------------------------------------------------------------------------#
def get_bucket_names_by_pattern(self, pattern):
""" return list of buckets with matching pattern
Parameter:
----------
pattern : string
Return:
-------
list with matches
"""
bucket_list = self.get_bucket_names() #list all buckets
match_list = [bucket_name for bucket_name in bucket_list if pattern in bucket_name]
self.my_logger.info('Found the following buckets: ' + str(match_list))
if len(match_list) == 0:
self.my_logger.error('No matching buckets for {0} were found.'.format(pattern))
else: # with pattern matching
match_list = [bucket_name for bucket_name in bucket_name_list if re.search(pattern, bucket_name)]
self.my_logger.info('Found the following buckets: ' + str(match_list))
if len(match_list) == 0:
self.my_logger.error('No matching buckets for {0} were found.'.format(pattern))
return match_list
return match_list
except Exception as e:
self.my_logger.error('Buckets could not be listed. ' + str(e))
#--------------------------------------------------------------------------------------------------------#
def create_s3_bucket(self, bucket_name, name_addition=True):
......@@ -339,55 +333,47 @@ class Loosolab_s3:
return bucket_name
#--------------------------------------------------------------------------------------------------------#
def delete_s3_bucket(self, bucket_name, only_delete_objects = False):
def emptie_s3_buckets(self, bucket_names, delete_bucket = False):
""" Delete and emptie bucket
Parameter:
----------
bucket_name : string
only_delete_objects : Bool
if True bucket will be emptie but not deleted
bucket_names : string or list of bucket names
delete_bucket : Bool
if True bucket will be deleted
"""
try:
bucket = self.session.Bucket(bucket_name)
except Exception as e:
self.my_logger.error('Bucket does not exist.' + str(e))
return
try:
bucket.objects.all().delete()
if only_delete_objects:
self.my_logger.info("Bucket " + bucket_name + " emptied, NOT deleted.")
return
except Exception as e: # todo Was ist wenn keine Objects da sind?
self.my_logger.error('Bucket Objects could not be deleted.' + str(e))
return
try:
bucket.delete()
self.my_logger.info("Bucket " + bucket_name + " deleted.")
except Exception as e:
self.my_logger.error('Bucket could not be deleted.' + str(e))
#--------------------------------------------------------------------------------------------------------#
def delete_s3_buckets(self, bucket_list, only_delete_objects = False):
""" Delete and emptie multiple buckets
Parameter:
----------
bucket_list : list of bucket names
only_delete_objects : Bool
if True buckets will be emptied but not deleted
"""
try:
for bucket_name in bucket_list:
self.delete_s3_bucket(bucket_name, only_delete_objects)
except Exception as e:
self.my_logger.error('Problem with list of Bucket names.' + str(e))
if isinstance(bucket_names, str):
bucket_list = [bucket_names]
else:
bucket_list = bucket_names
for bucket_name in bucket_list:
try:
bucket = self.session.Bucket(bucket_name)
except Exception as e:
self.my_logger.error('Bucket does not exist.' + str(e))
continue
try:
bucket.objects.all().delete()
if not delete_bucket:
self.my_logger.info("Bucket " + bucket_name + " emptied.")
continue
except Exception as e: # todo Was ist wenn keine Objects da sind?
self.my_logger.error('Bucket Objects could not be deleted.' + str(e))
continue
try:
bucket.delete()
self.my_logger.info("Bucket " + bucket_name + " deleted.")
except Exception as e:
self.my_logger.error('Bucket could not be deleted.' + str(e))
#--------------------------------------------------------------------------------------------------------#
#-------------- File Management -------------------------------------------------------------------------#
def get_object_names(self, bucket_name):
def get_object_names(self, bucket_name, pattern=""):
""" Get a list of all objects in bucket
Parameter:
----------
bucket_name : string
pattern : string
Returns:
--------
object_name_list : list
......@@ -397,61 +383,49 @@ class Loosolab_s3:
if not self.check_s3_bucket_ex(bucket_name):
self.my_logger.error('Bucket does not exist!')
return []
try:
try:
bucket = self.session.Bucket(bucket_name)
except Exception as e:
except Exception as e:
self.my_logger.error('Problem calling bucket. ' + str(e))
return
return
try:
for obj in bucket.objects.all():
object_name_list.append(obj.key)
return object_name_list
if not pattern:
return object_name_list
else:
match_list = [obj for obj in object_name_list if re.search(pattern, obj)]
self.my_logger.info('Found the following files: ' + str(match_list))
if len(match_list) == 0:
self.my_logger.error('No matching files for {0} were found in bucket {1}.'.format(pattern,bucket_name))
return match_list
except Exception as e:
self.my_logger.error('Objects in Bucket ' + bucket_name + 'could not be listed! ' + str(e))
#--------------------------------------------------------------------------------------------------------#
def get_object_names_by_pattern(self, bucket_name, pattern):
""" return list of files with matching pattern from bucket
Parameter:
----------
bucket_name : string
pattern : string
Return:
-------
list with matches
"""
object_list = self.get_object_names(bucket_name)
match_list = [obj for obj in object_list if pattern in obj]
self.my_logger.info('Found the following files: ' + str(match_list))
if len(match_list) == 0:
self.my_logger.error('No matching files for {0} were found in bucket {1}.'.format(pattern,bucket_name))
return match_list
#--------------------------------------------------------------------------------------------------------#
def upload_s3_objects(self, bucket_name, file_list):
def upload_s3_objects(self, bucket_name, file_list, compare=True):
"""Creating an s3 bucket.
Parameter:
----------
bucket_name : string
file_list : list
list of local filepaths
compare : boolean
should local and s3 file be compared?
Returns:
--------
Boolean
"""
modBool = False
if not self.check_s3_bucket_ex(bucket_name):
self.my_logger.error("Bucket for upload does not exist!" + str(e))
return
for local_file_name in file_list:
try:
file_name = os.path.basename(local_file_name)
modBool = self.compare_s3_etag(bucket_name, file_name, local_file_name)
if compare:
modBool = self.compare_s3_etag(bucket_name, file_name, local_file_name)
if not modBool:
if self.multipart_upload:
self.session.Bucket(bucket_name).upload_file(local_file_name, file_name)
......@@ -465,7 +439,7 @@ class Loosolab_s3:
return True
#--------------------------------------------------------------------------------------------------------#
def download_s3_objects(self, bucket_name, file_list, destination='.', timeout=0.1):
def download_s3_objects(self, bucket_name, file_list, destination='.', compare=True):
""" Download files from bucket
Parameter:
----------
......@@ -477,30 +451,21 @@ class Loosolab_s3:
how long to watch for file in minutes
"""
bucket = self.session.Bucket(bucket_name)
modBool = False
# check if exists
try:
for local_file in file_list:
downloaded = False
file_name = os.path.basename(local_file)
file_path = os.path.join(destination, local_file)
# check once a second if file is present, check for x minutes (->*60)
total_checks = int(60 * timeout)
modBool = self.compare_s3_etag(bucket_name, file_name, file_path)
for checks in range(total_checks -1):
if self.check_s3_object_ex(bucket_name, file_name) and not modBool:
if not os.path.exists(file_path):
os.makedirs(os.path.dirname(file_path))
self.my_logger.info('Created directory: ' + os.path.dirname(file_path))
if self.multipart_upload:
bucket.download_file(file_name, file_path)
else:
bucket.download_file(file_name, file_path, Config=self.transfer)
downloaded = True
break
time.sleep(1)
if not downloaded and modBool:
self.my_logger.info("File not downloaded.")
elif not downloaded:
self.my_logger.error("Could not download file " + file_name)
if compare:
modBool = self.compare_s3_etag(bucket_name, file_name, file_path)
if self.check_s3_object_ex(bucket_name, file_name) and not modBool:
if not os.path.exists(file_path):
os.makedirs(os.path.dirname(file_path))
self.my_logger.info('Created directory: ' + os.path.dirname(file_path))
if self.multipart_upload:
bucket.download_file(file_name, file_path)
else:
bucket.download_file(file_name, file_path, Config=self.transfer)
except Exception as e:
self.my_logger.error(str(e) + "Could not download file " + file_name)
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment