慢序列化过程Django rest框架
Posted
技术标签:
【中文标题】慢序列化过程Django rest框架【英文标题】：Slow Serializion process Django rest framework 【发布时间】：2020-12-22 16:38:19 【问题描述】：我正在为我的项目使用 django_rest_framework，但我遇到了问题。
models.py：
from django.db import models
class RelatedField3_2(models.Model):
name = models.CharField(max_length=100)
def __str__(self):
return self.name
class RelatedField3(models.Model):
name = models.CharField(max_length=100)
relfield3_2 = models.ForeignKey(RelatedField3_2, on_delete=models.CASCADE)
def __str__(self):
return self.name
class RelatedField2(models.Model):
name = models.CharField(max_length=100)
def __str__(self):
return self.name
class RelatedField1(models.Model):
name = models.CharField(max_length=200)
def __str__(self):
return self.name
class MainTable(models.Model):
owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
field1 = models.IntegerField()
field2 = models.IntegerField()
field3 = models.IntegerField()
field4 = models.DecimalField(max_digits=10, decimal_places=1)
field5 = models.IntegerField(null=True)
field6 = models.IntegerField()
relfield1 = models.ForeignKey(RelatedField1, on_delete=models.CASCADE)
relfield2 = models.ForeignKey(RelatedField2, on_delete=models.CASCADE)
relfield3 = models.ForeignKey(RelatedField3, on_delete=models.CASCADE)
serializers.py:
from rest_framework import serializers
from main.models import MainTable, RelatedField1, RelatedField2, RelatedField3
class MainTableRelatedFields(serializers.RelatedField):
def display_value(self, instance):
return instance
def to_representation(self, value):
return str(value)
def to_internal_value(self, data):
return self.queryset.model.objects.get(name=data)
class MainTableSerializerList(serializers.ListSerializer):
def create(self, validated_data):
records = [MainTable(**item) for item in validated_data]
return self.child.Meta.model.objects.bulk_create(records)
class MainTableSerializer(serializers.ModelSerializer):
class Meta:
model = MainTable
list_serializer_class = MainTableSerializerList
id = serializers.IntegerField(write_only=False, required=False)
owner = serializers.ReadOnlyField(source='owner.username')
relfield3 = PartnerPropRelatedFields(queryset=RelatedField3.objects.all())
relfield2 = PartnerPropRelatedFields(queryset=RelatedField2.objects.all())
relfield1 = PartnerPropRelatedFields(queryset=RelatedField1.objects.all())
def create(self, validated_data):
return self.Meta.model.objects.create(**validated_data)
views.py：
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework import viewsets
from rest_framework.permissions import IsAuthenticated
from .serializers import MainTableSerializer
class MainTableUploadView(viewsets.ModelViewSet):
permission_classes = (IsAuthenticated,)
@action(['post'], detail=False)
def upload_records(self, request, *args, **kwargs):
seriazlizer = MainTableSerializer(data=request.data, many=True)
seriazlizer.is_valid(raise_exception=True)
seriazlizer.save(owner=request.user)
return Response(seriazlizer.data)
send_json.py:
import requests
data_load_list = [
"field1": 1,
"field2": 2,
"field3": 3,
"field4": "field4",
"field5": 5,
"field6": "field6",
"relfield1": "test1",
"relfield2": "test2",
"relfield3" : "test3"
for i in range(1000)
]
load_list_response = requests.post(url=url_upload, headers='Authorization': f'Token TOKEN',
json=data_load_list)
当我启动 send_json.py 发送 1000 记录时，我只是得到错误：
RemoteDisconnected Traceback (most recent call last)
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
669 # Make the request on the httplib connection object.
--> 670 httplib_response = self._make_request(
671 conn,
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
425 # Otherwise it looks like a bug in the code.
--> 426 six.raise_from(e, None)
427 except (SocketTimeout, BaseSSLError, SocketError) as e:
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/packages/six.py in raise_from(value, from_value)
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
420 try:
--> 421 httplib_response = conn.getresponse()
422 except BaseException as e:
/usr/lib/python3.8/http/client.py in getresponse(self)
1331 try:
-> 1332 response.begin()
1333 except ConnectionError:
/usr/lib/python3.8/http/client.py in begin(self)
302 while True:
--> 303 version, status, reason = self._read_status()
304 if status != CONTINUE:
/usr/lib/python3.8/http/client.py in _read_status(self)
271 # sending a valid response.
--> 272 raise RemoteDisconnected("Remote end closed connection without"
273 " response")
RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
ProtocolError Traceback (most recent call last)
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
438 if not chunked:
--> 439 resp = conn.urlopen(
440 method=request.method,
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
725
--> 726 retries = retries.increment(
727 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
402 if read is False or not self._is_method_retryable(method):
--> 403 raise six.reraise(type(error), error, _stacktrace)
404 elif read is not None:
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
733 if value.__traceback__ is not tb:
--> 734 raise value.with_traceback(tb)
735 raise value
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
669 # Make the request on the httplib connection object.
--> 670 httplib_response = self._make_request(
671 conn,
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
425 # Otherwise it looks like a bug in the code.
--> 426 six.raise_from(e, None)
427 except (SocketTimeout, BaseSSLError, SocketError) as e:
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/packages/six.py in raise_from(value, from_value)
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
420 try:
--> 421 httplib_response = conn.getresponse()
422 except BaseException as e:
/usr/lib/python3.8/http/client.py in getresponse(self)
1331 try:
-> 1332 response.begin()
1333 except ConnectionError:
/usr/lib/python3.8/http/client.py in begin(self)
302 while True:
--> 303 version, status, reason = self._read_status()
304 if status != CONTINUE:
/usr/lib/python3.8/http/client.py in _read_status(self)
271 # sending a valid response.
--> 272 raise RemoteDisconnected("Remote end closed connection without"
273 " response")
ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
<timed exec> in <module>
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/requests/api.py in post(url, data, json, **kwargs)
117 """
118
--> 119 return request('post', url, data=data, json=json, **kwargs)
120
121
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/requests/api.py in request(method, url, **kwargs)
59 # cases, and look like a memory leak in others.
60 with sessions.Session() as session:
---> 61 return session.request(method=method, url=url, **kwargs)
62
63
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
528
529 send_kwargs.update(settings)
--> 530 resp = self.send(prep, **send_kwargs)
531
532 return resp
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/requests/sessions.py in send(self, request, **kwargs)
641
642 # Send the request
--> 643 r = adapter.send(request, **kwargs)
644
645 # Total elapsed time of the request (approximately)
~/.local/share/virtualenvs/DKykH6t1/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
496
497 except (ProtocolError, socket.error) as err:
--> 498 raise ConnectionError(err, request=request)
499
500 except MaxRetryError as e:
ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
当我启动 send_json.py 以发送 100 条记录时，数据被加载到数据库中并返回代码 200。这需要 15 秒。
我可以如何优化我的代码，以便我可以将 1000 多条记录发送到数据库并在响应正文中快速获取信息？
【问题讨论】：
您的请求是立即终止还是在一段时间后终止？还向我们展示服务器端的日志。 请求会在一段时间后终止（等待 10-20 秒）。服务器端日志不显示有关此类请求的任何信息。 @AviKKi 我可以从某处获取替代日志，或者记录此类请求吗？ 这可能是客户端超时，尝试设置更大的超时requests.post(url=url_upload, headers='Authorization': f'Token TOKEN',json=data_load_list, timeout=3600)，将这么多条目添加到数据库需要时间这是正常的。稍后我会用一些优化技术来回答。 PS：你想在你的 Django 数据库中加载大量数据吗？您不应该通过请求发送它，而应该使用自定义管理命令。让我知道你的用例，我会给你一些有用的代码 sn-p。
@AviKKi 感谢您的建议。增加客户端请求的超时时间并没有帮助，10-20秒后请求仍然中断并出现相同的错误。是的，我的目标是通过 post 请求 将大量数据精确地加载到服务器，同时在响应中接收的不是相关字段的 id，而是它们的名称
【参考方案1】：
对我有帮助的解决方案（请参阅拉取请求中的更改）：
https://github.com/hax2000/Relaed-Fields-API/pull/1/files
【讨论】：
以上是关于慢序列化过程Django rest框架的主要内容，如果未能解决你的问题，请参考以下文章
基于Django的Rest Framework框架的序列化组件