Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Discuss Data Project
Discuss Data
Commits
6ba99e1f
Commit
6ba99e1f
authored
May 25, 2021
by
felix.herrmann
Browse files
Merge branch '402-upload-big-files-to-dariah' into 'master'
Resolve "upload big files to dariah" Closes
#402
See merge request
!402
parents
fe7ce10c
2f8935a2
Changes
4
Pipelines
7
Hide whitespace changes
Inline
Side-by-side
discuss_data/dddatasets/views/prep.py
View file @
6ba99e1f
...
...
@@ -1484,9 +1484,9 @@ def edit_publish_final_publish(request, ds_uuid):
publisher
=
Publisher
(
t
,
ds
)
# get collection from session
collection
=
request
.
session
.
get
(
"collection"
,
{})
logger
.
error
(
"[DHREP PUBLISHER] collection: %s"
,
str
(
collection
))
logger
.
debug
(
"[DHREP PUBLISHER] collection: %s"
,
str
(
collection
))
status
=
publisher
.
publish_status
(
collection
)
logger
.
error
(
"[DHREP PUBLISHER] status"
,
status
)
logger
.
debug
(
"[DHREP PUBLISHER] status
: %s
"
,
status
)
# an API implementation flaw makes the "processStatus" entry missing sometimes; i suppose this is in the state of QUEUING
try
:
...
...
@@ -1523,7 +1523,7 @@ def edit_publish_final_publish(request, ds_uuid):
request
.
session
.
pop
(
"collection"
)
publisher
.
update_dataset
(
collection
,
status
)
# also, make DD publication when finished
logger
.
error
(
"[DHREP PUBLISHER] finished, starting ds.publish"
,
status
)
logger
.
debug
(
"[DHREP PUBLISHER] finished, starting ds.publish
: %s
"
,
status
)
ds
.
publish
(
request
.
user
)
messages
.
success
(
request
,
...
...
@@ -1534,7 +1534,7 @@ def edit_publish_final_publish(request, ds_uuid):
if
process_status
==
"FAILED"
:
response
[
"X-IC-CancelPolling"
]
=
"true"
logger
.
error
(
"[DHREP PUBLISHER] FAILED"
,
status
)
logger
.
error
(
"[DHREP PUBLISHER] FAILED
: %s
"
,
status
)
messages
.
error
(
request
,
_
(
"An Error occured. The dataset has not been published."
)
)
...
...
@@ -1648,7 +1648,7 @@ def edit_publish_request(request, ds_uuid):
if
request
.
method
==
"POST"
:
# run checks
logger
.
error
(
"POST publish request"
)
logger
.
debug
(
"POST publish request"
)
check_error
,
check_list
=
ds
.
perform_checks
()
logger
.
error
(
check_error
)
if
check_error
:
...
...
@@ -1657,9 +1657,9 @@ def edit_publish_request(request, ds_uuid):
logger
.
error
(
error_string
)
messages
.
error
(
request
,
error_string
)
else
:
logger
.
error
(
"start request_publication"
)
logger
.
debug
(
"start request_publication"
)
publish_message
=
ds
.
request_publication
(
request
.
user
)
logger
.
error
(
"end request_publication"
)
logger
.
debug
(
"end request_publication"
)
return
edit_versions
(
request
,
ds_uuid
)
else
:
publish_message
=
None
...
...
discuss_data/dhrep/publisher.py
View file @
6ba99e1f
...
...
@@ -45,8 +45,8 @@ class Publisher:
validator
=
URLValidator
([
"https"
])
try
:
validator
(
settings
.
PUBLISH_URL
)
except
ValidationError
:
raise
ImproperlyConfigured
except
ValidationError
as
e
:
raise
ImproperlyConfigured
from
e
publish_url
=
settings
.
PUBLISH_URL
if
not
publish_url
.
endswith
(
"/"
):
...
...
@@ -143,7 +143,7 @@ class Publisher:
"collection.ttl"
,
{
"storage_id"
:
storage_id
,
"dataset"
:
dataset
,
"datafiles"
:
datafiles
},
)
logger
.
error
(
"[COLLECTION_RDF]"
,
turtle
)
logger
.
debug
(
"[COLLECTION_RDF]
: %s
"
,
turtle
)
return
turtle
...
...
@@ -198,8 +198,8 @@ class Publisher:
},
)
logger
.
error
(
"status-text: %s"
,
response
.
text
)
logger
.
error
(
"status-status: %s"
,
response
.
status_code
)
logger
.
debug
(
"status-text: %s"
,
response
.
text
)
logger
.
debug
(
"status-status: %s"
,
response
.
status_code
)
if
response
.
status_code
!=
200
:
raise
PublisherError
(
...
...
@@ -287,11 +287,11 @@ class Publisher:
for
file
in
files
.
keys
():
try
:
dariah_storage
.
get
(
self
.
token
,
file
)
#
logger.debug("DARIAH UPLOAD STATUS:
{}
finished"
.
format(file))
logger
.
debug
(
"DARIAH UPLOAD STATUS:
%s
finished"
,
format
(
file
))
except
StorageException
as
e
:
status
[
file
]
=
False
# do not show complete traceback in debug logging
logger
.
error
(
"[[DARIAH UPLOAD STATUS]]:
{}"
.
format
(
e
.
args
[
0
]))
logger
.
debug
(
"[[DARIAH UPLOAD STATUS]]:
%s "
,
format
(
e
.
args
[
0
]))
else
:
status
[
file
]
=
True
...
...
discuss_data/dhrep/services.py
View file @
6ba99e1f
...
...
@@ -35,7 +35,7 @@ def update(token: Token, storage_id: str, file, content_type=None) -> None:
content
=
file
# https://docs.djangoproject.com/en/2.2/ref/models/fields/#django.db.models.fields.files.FieldFile
content_type
=
"text/plain; charset=utf-8"
with
content
.
open
()
as
stream
:
with
content
.
open
(
mode
=
"rb"
)
as
stream
:
response
=
requests
.
put
(
settings
.
STORAGE_URL
+
storage_id
,
headers
=
{
...
...
discuss_data/dhrep/storage.py
View file @
6ba99e1f
...
...
@@ -23,8 +23,8 @@ class Storage:
validator
=
URLValidator
([
"https"
])
try
:
validator
(
settings
.
STORAGE_URL
)
except
ValidationError
:
raise
ImproperlyConfigured
except
ValidationError
as
e
:
raise
ImproperlyConfigured
from
e
storage_url
=
settings
.
STORAGE_URL
if
not
storage_url
.
endswith
(
"/"
):
...
...
@@ -47,10 +47,10 @@ class Storage:
raise
StorageException
(
"Not yet uploaded "
+
storage_id
+
": "
+
response
.
text
+
" - "
+
". "
+
str
(
response
.
status_code
)
+
": "
+
response
.
reason
)
return
response
...
...
@@ -86,7 +86,7 @@ class Storage:
return
storage_id
def
update
(
self
,
token
:
Token
,
storage_id
:
str
,
content
)
->
None
:
"""Update an object in dariahstorage
"""Update an object in dariahstorage
(DEPRECATED)
:param token: authentication token
:type token: Token
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment