elasticsearch ConnectionTimeout
when clicking pubdd (discuss data publication, last step in publication wizard), user is redirected to the 500 view
this occurs occasionally
WARNING 2020-07-14 10:56:24,194 base 93 140298570059080 POST http://elasticsearch:9200/_bulk?refresh=true [status:N/A request:10.012s]
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.7/site-packages/sentry_sdk/integrations/stdlib.py", line 102, in getresponse
rv = real_getresponse(self, *args, **kwargs)
File "/usr/local/lib/python3.7/http/client.py", line 1354, in getresponse
response.begin()
File "/usr/local/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.7/http/client.py", line 267, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py", line 233, in perform_request
method, url, body, retries=Retry(False), headers=request_headers, **kw
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 725, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 379, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.7/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 428, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 336, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='elasticsearch', port=9200): Read timed out. (read timeout=10)
ERROR 2020-07-14 10:56:32,252 log 93 140298570059080 Internal Server Error: /dataset/prep/09ebb8e2-da85-4058-b0b6-5faa0cb4777f/edit/publish/final/pubdd
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.7/site-packages/sentry_sdk/integrations/stdlib.py", line 102, in getresponse
rv = real_getresponse(self, *args, **kwargs)
File "/usr/local/lib/python3.7/http/client.py", line 1354, in getresponse
response.begin()
File "/usr/local/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.7/http/client.py", line 267, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py", line 233, in perform_request
method, url, body, retries=Retry(False), headers=request_headers, **kw
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 725, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 379, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.7/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 428, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 336, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='elasticsearch', port=9200): Read timed out. (read timeout=10)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/exception.py", line 34, in inner
response = get_response(request)
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/base.py", line 115, in _get_response
response = self.process_exception_by_middleware(e, request)
File "/usr/local/lib/python3.7/site-packages/django/core/handlers/base.py", line 113, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/usr/local/lib/python3.7/contextlib.py", line 74, in inner
return func(*args, **kwds)
File "/usr/local/lib/python3.7/site-packages/django/contrib/auth/decorators.py", line 21, in _wrapped_view
return view_func(request, *args, **kwargs)
File "/app/discuss_data/dddatasets/views_prep.py", line 1311, in prep_dataset_edit_publish_final_pubdd
ds.publish(request.user)
File "/app/discuss_data/dddatasets/models.py", line 1080, in publish
self.save()
File "/app/discuss_data/dddatasets/models.py", line 926, in save
models.Model.save(self, *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 741, in save
force_update=force_update, update_fields=update_fields)
File "/usr/local/lib/python3.7/site-packages/django/db/models/base.py", line 790, in save_base
update_fields=update_fields, raw=raw, using=using,
File "/usr/local/lib/python3.7/site-packages/django/dispatch/dispatcher.py", line 175, in send
for receiver in self._live_receivers(sender)
File "/usr/local/lib/python3.7/site-packages/django/dispatch/dispatcher.py", line 175, in <listcomp>
for receiver in self._live_receivers(sender)
File "/usr/local/lib/python3.7/site-packages/django_elasticsearch_dsl/signals.py", line 57, in handle_save
registry.update(instance)
File "/usr/local/lib/python3.7/site-packages/django_elasticsearch_dsl/registries.py", line 141, in update
doc().update(instance, **kwargs)
File "/app/discuss_data/core/documents.py", line 97, in update
return super().update(self.filter_instance(instance), refresh, action, **kwargs)
File "/usr/local/lib/python3.7/site-packages/django_elasticsearch_dsl/documents.py", line 207, in update
**kwargs
File "/usr/local/lib/python3.7/site-packages/django_elasticsearch_dsl/documents.py", line 188, in _bulk
return self.bulk(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/django_elasticsearch_dsl/documents.py", line 146, in bulk
return bulk(client=self._get_connection(), actions=actions, **kwargs)
File "/usr/local/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 360, in bulk
for ok, item in streaming_bulk(client, actions, *args, **kwargs):
File "/usr/local/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 290, in streaming_bulk
**kwargs
File "/usr/local/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 217, in _process_bulk_chunk
for item in gen:
File "/usr/local/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 166, in _process_bulk_chunk_error
raise error
File "/usr/local/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 205, in _process_bulk_chunk
resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/elasticsearch/client/utils.py", line 139, in _wrapped
return func(*args, params=params, headers=headers, **kwargs)
File "/usr/local/lib/python3.7/site-packages/elasticsearch/client/__init__.py", line 433, in bulk
body=body,
File "/usr/local/lib/python3.7/site-packages/elasticsearch/transport.py", line 352, in perform_request
timeout=timeout,
File "/usr/local/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py", line 244, in perform_request
raise ConnectionTimeout("TIMEOUT", str(e), e)
elasticsearch.exceptions.ConnectionTimeout: ConnectionTimeout caused by - ReadTimeoutError(HTTPConnectionPool(host='elasticsearch', port=9200): Read timed out. (read timeout=10))
[2020-07-14 10:56:35 +0000] [91] [CRITICAL] WORKER TIMEOUT (pid:93)
[2020-07-14 10:56:36 +0000] [93] [INFO] Worker exiting (pid: 93)
[2020-07-14 10:56:43 +0000] [401] [INFO] Booting worker with pid: 401
[2020-07-14 10:57:11 +0000] [91] [CRITICAL] WORKER TIMEOUT (pid:401)
[2020-07-14 10:57:11 +0000] [401] [INFO] Worker exiting (pid: 401)
[2020-07-14 10:57:11 +0000] [402] [INFO] Booting worker with pid: 402
Edited by Stefan Hynek