Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Discuss Data Project
Discuss Data
Commits
6ba99e1f
Commit
6ba99e1f
authored
May 25, 2021
by
felix.herrmann
Browse files
Merge branch '402-upload-big-files-to-dariah' into 'master'
Resolve "upload big files to dariah" Closes
#402
See merge request
!402
parents
fe7ce10c
2f8935a2
Changes
4
Pipelines
7
Hide whitespace changes
Inline
Side-by-side
discuss_data/dddatasets/views/prep.py
View file @
6ba99e1f
...
...
@@ -1484,9 +1484,9 @@ def edit_publish_final_publish(request, ds_uuid):
publisher
=
Publisher
(
t
,
ds
)
# get collection from session
collection
=
request
.
session
.
get
(
"collection"
,
{})
logger
.
error
(
"[DHREP PUBLISHER] collection: %s"
,
str
(
collection
))
logger
.
debug
(
"[DHREP PUBLISHER] collection: %s"
,
str
(
collection
))
status
=
publisher
.
publish_status
(
collection
)
logger
.
error
(
"[DHREP PUBLISHER] status"
,
status
)
logger
.
debug
(
"[DHREP PUBLISHER] status
: %s
"
,
status
)
# an API implementation flaw makes the "processStatus" entry missing sometimes; i suppose this is in the state of QUEUING
try
:
...
...
@@ -1523,7 +1523,7 @@ def edit_publish_final_publish(request, ds_uuid):
request
.
session
.
pop
(
"collection"
)
publisher
.
update_dataset
(
collection
,
status
)
# also, make DD publication when finished
logger
.
error
(
"[DHREP PUBLISHER] finished, starting ds.publish"
,
status
)
logger
.
debug
(
"[DHREP PUBLISHER] finished, starting ds.publish
: %s
"
,
status
)
ds
.
publish
(
request
.
user
)
messages
.
success
(
request
,
...
...
@@ -1534,7 +1534,7 @@ def edit_publish_final_publish(request, ds_uuid):
if
process_status
==
"FAILED"
:
response
[
"X-IC-CancelPolling"
]
=
"true"
logger
.
error
(
"[DHREP PUBLISHER] FAILED"
,
status
)
logger
.
error
(
"[DHREP PUBLISHER] FAILED
: %s
"
,
status
)
messages
.
error
(
request
,
_
(
"An Error occured. The dataset has not been published."
)
)
...
...
@@ -1648,7 +1648,7 @@ def edit_publish_request(request, ds_uuid):
if
request
.
method
==
"POST"
:
# run checks
logger
.
error
(
"POST publish request"
)
logger
.
debug
(
"POST publish request"
)
check_error
,
check_list
=
ds
.
perform_checks
()
logger
.
error
(
check_error
)
if
check_error
:
...
...
@@ -1657,9 +1657,9 @@ def edit_publish_request(request, ds_uuid):
logger
.
error
(
error_string
)
messages
.
error
(
request
,
error_string
)
else
:
logger
.
error
(
"start request_publication"
)
logger
.
debug
(
"start request_publication"
)
publish_message
=
ds
.
request_publication
(
request
.
user
)
logger
.
error
(
"end request_publication"
)
logger
.
debug
(
"end request_publication"
)
return
edit_versions
(
request
,
ds_uuid
)
else
:
publish_message
=
None
...
...
discuss_data/dhrep/publisher.py
View file @
6ba99e1f
...
...
@@ -45,8 +45,8 @@ class Publisher:
validator
=
URLValidator
([
"https"
])
try
:
validator
(
settings
.
PUBLISH_URL
)
except
ValidationError
:
raise
ImproperlyConfigured
except
ValidationError
as
e
:
raise
ImproperlyConfigured
from
e
publish_url
=
settings
.
PUBLISH_URL
if
not
publish_url
.
endswith
(
"/"
):
...
...
@@ -143,7 +143,7 @@ class Publisher:
"collection.ttl"
,
{
"storage_id"
:
storage_id
,
"dataset"
:
dataset
,
"datafiles"
:
datafiles
},
)
logger
.
error
(
"[COLLECTION_RDF]"
,
turtle
)
logger
.
debug
(
"[COLLECTION_RDF]
: %s
"
,
turtle
)
return
turtle
...
...
@@ -198,8 +198,8 @@ class Publisher:
},
)
logger
.
error
(
"status-text: %s"
,
response
.
text
)
logger
.
error
(
"status-status: %s"
,
response
.
status_code
)
logger
.
debug
(
"status-text: %s"
,
response
.
text
)
logger
.
debug
(
"status-status: %s"
,
response
.
status_code
)
if
response
.
status_code
!=
200
:
raise
PublisherError
(
...
...
@@ -287,11 +287,11 @@ class Publisher:
for
file
in
files
.
keys
():
try
:
dariah_storage
.
get
(
self
.
token
,
file
)
#
logger.debug("DARIAH UPLOAD STATUS:
{}
finished"
.
format(file))
logger
.
debug
(
"DARIAH UPLOAD STATUS:
%s
finished"
,
format
(
file
))
except
StorageException
as
e
:
status
[
file
]
=
False
# do not show complete traceback in debug logging
logger
.
error
(
"[[DARIAH UPLOAD STATUS]]:
{}"
.
format
(
e
.
args
[
0
]))
logger
.
debug
(
"[[DARIAH UPLOAD STATUS]]:
%s "
,
format
(
e
.
args
[
0
]))
else
:
status
[
file
]
=
True
...
...
discuss_data/dhrep/services.py
View file @
6ba99e1f
...
...
@@ -35,7 +35,7 @@ def update(token: Token, storage_id: str, file, content_type=None) -> None:
content
=
file
# https://docs.djangoproject.com/en/2.2/ref/models/fields/#django.db.models.fields.files.FieldFile
content_type
=
"text/plain; charset=utf-8"
with
content
.
open
()
as
stream
:
with
content
.
open
(
mode
=
"rb"
)
as
stream
:
response
=
requests
.
put
(
settings
.
STORAGE_URL
+
storage_id
,
headers
=
{
...
...
discuss_data/dhrep/storage.py
View file @
6ba99e1f
...
...
@@ -23,8 +23,8 @@ class Storage:
validator
=
URLValidator
([
"https"
])
try
:
validator
(
settings
.
STORAGE_URL
)
except
ValidationError
:
raise
ImproperlyConfigured
except
ValidationError
as
e
:
raise
ImproperlyConfigured
from
e
storage_url
=
settings
.
STORAGE_URL
if
not
storage_url
.
endswith
(
"/"
):
...
...
@@ -47,10 +47,10 @@ class Storage:
raise
StorageException
(
"Not yet uploaded "
+
storage_id
+
": "
+
response
.
text
+
" - "
+
". "
+
str
(
response
.
status_code
)
+
": "
+
response
.
reason
)
return
response
...
...
@@ -86,7 +86,7 @@ class Storage:
return
storage_id
def
update
(
self
,
token
:
Token
,
storage_id
:
str
,
content
)
->
None
:
"""Update an object in dariahstorage
"""Update an object in dariahstorage
(DEPRECATED)
:param token: authentication token
:type token: Token
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment