diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000000..a6a922a221b
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,17 @@
+version: 2
+updates:
+ - package-ecosystem: "pip"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ timezone: "Asia/Shanghai"
+ day: "friday"
+ target-branch: "v2"
+ groups:
+ python-dependencies:
+ patterns:
+ - "*"
+# ignore:
+# - dependency-name: "pymupdf"
+# versions: ["*"]
+
diff --git a/.github/workflows/build-and-push-python-pg.yml b/.github/workflows/build-and-push-python-pg.yml
index bc4dc3f2c77..4640f5edbd0 100644
--- a/.github/workflows/build-and-push-python-pg.yml
+++ b/.github/workflows/build-and-push-python-pg.yml
@@ -33,13 +33,13 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
with:
- ref: main
+ ref: v1
- name: Prepare
id: prepare
run: |
DOCKER_IMAGE=ghcr.io/1panel-dev/maxkb-python-pg
DOCKER_PLATFORMS=${{ github.event.inputs.architecture }}
- TAG_NAME=python3.11-pg15.8
+ TAG_NAME=python3.11-pg15.14
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
echo ::set-output name=docker_image::${DOCKER_IMAGE}
echo ::set-output name=version::${TAG_NAME}
diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml
index 26d2b86d297..1e1daf2696c 100644
--- a/.github/workflows/build-and-push.yml
+++ b/.github/workflows/build-and-push.yml
@@ -7,7 +7,7 @@ on:
inputs:
dockerImageTag:
description: 'Image Tag'
- default: 'v1.10.3-dev'
+ default: 'v1.10.7-dev'
required: true
dockerImageTagWithLatest:
description: '是否发布latest tag（正式发版时选择，测试版本切勿选择）'
@@ -36,7 +36,7 @@ on:
jobs:
build-and-push-to-fit2cloud-registry:
if: ${{ contains(github.event.inputs.registry, 'fit2cloud') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
steps:
- name: Check Disk Space
run: df -h
@@ -52,10 +52,6 @@ jobs:
swap-storage: true
- name: Check Disk Space
run: df -h
- - name: Set Swap Space
- uses: pierotofy/set-swap-space@master
- with:
- swap-size-gb: 8
- name: Checkout
uses: actions/checkout@v4
with:
@@ -68,24 +64,17 @@ jobs:
TAG_NAME=${{ github.event.inputs.dockerImageTag }}
TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then
- DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
+ DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*}"
else
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
fi
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
- --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=${GITHUB_SHA::8} --no-cache \
+ --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
${DOCKER_IMAGE_TAGS} .
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- with:
- # Until https://github.com/tonistiigi/binfmt/issues/215
- image: tonistiigi/binfmt:qemu-v7.0.0-28
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- with:
- buildkitd-config-inline: |
- [worker.oci]
- max-parallelism = 1
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
@@ -100,11 +89,12 @@ jobs:
password: ${{ secrets.FIT2CLOUD_REGISTRY_PASSWORD }}
- name: Docker Buildx (build-and-push)
run: |
+ sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches && free -m
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile
build-and-push-to-dockerhub:
if: ${{ contains(github.event.inputs.registry, 'dockerhub') }}
- runs-on: ubuntu-22.04
+ runs-on: ubuntu-latest
steps:
- name: Check Disk Space
run: df -h
@@ -120,10 +110,6 @@ jobs:
swap-storage: true
- name: Check Disk Space
run: df -h
- - name: Set Swap Space
- uses: pierotofy/set-swap-space@master
- with:
- swap-size-gb: 8
- name: Checkout
uses: actions/checkout@v4
with:
@@ -136,24 +122,17 @@ jobs:
TAG_NAME=${{ github.event.inputs.dockerImageTag }}
TAG_NAME_WITH_LATEST=${{ github.event.inputs.dockerImageTagWithLatest }}
if [[ ${TAG_NAME_WITH_LATEST} == 'true' ]]; then
- DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:latest"
+ DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME} --tag ${DOCKER_IMAGE}:${TAG_NAME%%.*}"
else
DOCKER_IMAGE_TAGS="--tag ${DOCKER_IMAGE}:${TAG_NAME}"
fi
echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} --memory-swap -1 \
- --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=${GITHUB_SHA::8} --no-cache \
+ --build-arg DOCKER_IMAGE_TAG=${{ github.event.inputs.dockerImageTag }} --build-arg BUILD_AT=$(TZ=Asia/Shanghai date +'%Y-%m-%dT%H:%M') --build-arg GITHUB_COMMIT=`git rev-parse --short HEAD` --no-cache \
${DOCKER_IMAGE_TAGS} .
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- with:
- # Until https://github.com/tonistiigi/binfmt/issues/215
- image: tonistiigi/binfmt:qemu-v7.0.0-28
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- with:
- buildkitd-config-inline: |
- [worker.oci]
- max-parallelism = 1
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
@@ -167,4 +146,5 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Docker Buildx (build-and-push)
run: |
+ sudo sync && echo 3 | sudo tee /proc/sys/vm/drop_caches && free -m
docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} -f installer/Dockerfile
diff --git a/README.md b/README.md
index cfe819e56ff..b4a925edb64 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
-Ready-to-use AI Chatbot
+Open-source platform for building enterprise-grade agents
+强大易用的企业级智能体平台
Feature
diff --git a/README_CN.md b/README_CN.md
index e55150902ea..07fa00ea4e6 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -1,25 +1,25 @@
-基于大模型和 RAG 的知识库问答系统
-Ready-to-use, flexible RAG Chatbot
+强大易用的企业级智能体平台
Called MCP Tool: %s
-```json
%s
-```
+
"""
+tool_message_json_template = """
+```json
+%s
+```
+"""
+
+
+def generate_tool_message_template(name, context):
+ if '```' in context:
+ return tool_message_template % (name, context)
+ else:
+ return tool_message_template % (name, tool_message_json_template % (context))
+
+
def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str,
reasoning_content: str):
chat_model = node_variable.get('chat_model')
@@ -102,19 +114,19 @@ def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INo
_write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content)
-
async def _yield_mcp_response(chat_model, message_list, mcp_servers):
async with MultiServerMCPClient(json.loads(mcp_servers)) as client:
agent = create_react_agent(chat_model, client.get_tools())
response = agent.astream({"messages": message_list}, stream_mode='messages')
async for chunk in response:
if isinstance(chunk[0], ToolMessage):
- content = tool_message_template % (chunk[0].name, chunk[0].content)
+ content = generate_tool_message_template(chunk[0].name, chunk[0].content)
chunk[0].content = content
yield chunk[0]
if isinstance(chunk[0], AIMessageChunk):
yield chunk[0]
+
def mcp_response_generator(chat_model, message_list, mcp_servers):
loop = asyncio.new_event_loop()
try:
@@ -130,6 +142,7 @@ def mcp_response_generator(chat_model, message_list, mcp_servers):
finally:
loop.close()
+
async def anext_async(agen):
return await agen.__anext__()
@@ -186,7 +199,9 @@ def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
self.context['question'] = details.get('question')
self.context['reasoning_content'] = details.get('reasoning_content')
- self.answer_text = details.get('answer')
+ self.context['model_setting'] = details.get('model_setting')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, chat_record_id,
model_params_setting=None,
@@ -216,7 +231,7 @@ def execute(self, model_id, system, prompt, dialogue_number, history_chat_record
message_list = self.generate_message_list(system, prompt, history_message)
self.context['message_list'] = message_list
- if mcp_enable and mcp_servers is not None:
+ if mcp_enable and mcp_servers is not None and '"stdio"' not in mcp_servers:
r = mcp_response_generator(chat_model, message_list, mcp_servers)
return NodeResult(
{'result': r, 'chat_model': chat_model, 'message_list': message_list,
@@ -271,6 +286,7 @@ def get_details(self, index: int, **kwargs):
"index": index,
'run_time': self.context.get('run_time'),
'system': self.context.get('system'),
+ 'model_setting': self.context.get('model_setting'),
'history_message': [{'content': message.content, 'role': message.type} for message in
(self.context.get('history_message') if self.context.get(
'history_message') is not None else [])],
diff --git a/apps/application/flow/step_node/application_node/impl/base_application_node.py b/apps/application/flow/step_node/application_node/impl/base_application_node.py
index d962f7163bb..95445f45612 100644
--- a/apps/application/flow/step_node/application_node/impl/base_application_node.py
+++ b/apps/application/flow/step_node/application_node/impl/base_application_node.py
@@ -168,7 +168,8 @@ def save_context(self, details, workflow_manage):
self.context['question'] = details.get('question')
self.context['type'] = details.get('type')
self.context['reasoning_content'] = details.get('reasoning_content')
- self.answer_text = details.get('answer')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type,
app_document_list=None, app_image_list=None, app_audio_list=None, child_node=None, node_data=None,
@@ -178,7 +179,8 @@ def execute(self, application_id, message, chat_id, chat_record_id, stream, re_c
current_chat_id = string_to_uuid(chat_id + application_id)
Chat.objects.get_or_create(id=current_chat_id, defaults={
'application_id': application_id,
- 'abstract': message[0:1024]
+ 'abstract': message[0:1024],
+ 'client_id': client_id,
})
if app_document_list is None:
app_document_list = []
diff --git a/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py b/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py
index 6a51edd6bae..1d3115e4c67 100644
--- a/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py
+++ b/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py
@@ -15,7 +15,9 @@
class BaseReplyNode(IReplyNode):
def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
- self.answer_text = details.get('answer')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
+
def execute(self, reply_type, stream, fields=None, content=None, **kwargs) -> NodeResult:
if reply_type == 'referencing':
result = self.get_reference_content(fields)
diff --git a/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py b/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py
index 6ddcb6e2fca..0c4d09bce5c 100644
--- a/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py
+++ b/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py
@@ -66,7 +66,7 @@ def save_image(image_list):
for doc in document:
file = QuerySet(File).filter(id=doc['file_id']).first()
- buffer = io.BytesIO(file.get_byte().tobytes())
+ buffer = io.BytesIO(file.get_byte())
buffer.name = doc['name'] # this is the important line
for split_handle in (parse_table_handle_list + split_handles):
diff --git a/apps/application/flow/step_node/form_node/impl/base_form_node.py b/apps/application/flow/step_node/form_node/impl/base_form_node.py
index 7cbbe9cc1d4..dcf35dd3cfd 100644
--- a/apps/application/flow/step_node/form_node/impl/base_form_node.py
+++ b/apps/application/flow/step_node/form_node/impl/base_form_node.py
@@ -38,7 +38,8 @@ def save_context(self, details, workflow_manage):
self.context['start_time'] = details.get('start_time')
self.context['form_data'] = form_data
self.context['is_submit'] = details.get('is_submit')
- self.answer_text = details.get('result')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('result')
if form_data is not None:
for key in form_data:
self.context[key] = form_data[key]
@@ -70,7 +71,7 @@ def get_answer_list(self) -> List[Answer] | None:
"chat_record_id": self.flow_params_serializer.data.get("chat_record_id"),
'form_data': self.context.get('form_data', {}),
"is_submit": self.context.get("is_submit", False)}
- form = f'{json.dumps(form_setting,ensure_ascii=False)} '
+ form = f'{json.dumps(form_setting, ensure_ascii=False)} '
context = self.workflow_manage.get_workflow_content()
form_content_format = self.workflow_manage.reset_prompt(form_content_format)
prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2')
@@ -85,7 +86,7 @@ def get_details(self, index: int, **kwargs):
"chat_record_id": self.flow_params_serializer.data.get("chat_record_id"),
'form_data': self.context.get('form_data', {}),
"is_submit": self.context.get("is_submit", False)}
- form = f'{json.dumps(form_setting,ensure_ascii=False)} '
+ form = f'{json.dumps(form_setting, ensure_ascii=False)} '
context = self.workflow_manage.get_workflow_content()
form_content_format = self.workflow_manage.reset_prompt(form_content_format)
prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2')
diff --git a/apps/application/flow/step_node/function_lib_node/impl/base_function_lib_node.py b/apps/application/flow/step_node/function_lib_node/impl/base_function_lib_node.py
index d21424f750d..0678b81243c 100644
--- a/apps/application/flow/step_node/function_lib_node/impl/base_function_lib_node.py
+++ b/apps/application/flow/step_node/function_lib_node/impl/base_function_lib_node.py
@@ -45,6 +45,8 @@ def get_field_value(debug_field_list, name, is_required):
def valid_reference_value(_type, value, name):
+ if value is None:
+ return
if _type == 'int':
instance_type = int | float
elif _type == 'float':
@@ -65,15 +67,22 @@ def valid_reference_value(_type, value, name):
def convert_value(name: str, value, _type, is_required, source, node):
- if not is_required and value is None:
+ if not is_required and (value is None or (isinstance(value, str) and len(value) == 0)):
return None
if not is_required and source == 'reference' and (value is None or len(value) == 0):
return None
if source == 'reference':
+ if value and isinstance(value, list) and len(value) == 0:
+ if not is_required:
+ return None
+ else:
+ raise Exception(f"字段:{name}类型:{_type}值:{value}必填参数")
value = node.workflow_manage.get_reference_field(
value[0],
value[1:])
valid_reference_value(_type, value, name)
+ if value is None:
+ return None
if _type == 'int':
return int(value)
if _type == 'float':
@@ -113,7 +122,8 @@ def valid_function(function_lib, user_id):
class BaseFunctionLibNodeNode(IFunctionLibNode):
def save_context(self, details, workflow_manage):
self.context['result'] = details.get('result')
- self.answer_text = str(details.get('result'))
+ if self.node_params.get('is_result'):
+ self.answer_text = str(details.get('result'))
def execute(self, function_lib_id, input_field_list, **kwargs) -> NodeResult:
function_lib = QuerySet(FunctionLib).filter(id=function_lib_id).first()
diff --git a/apps/application/flow/step_node/function_node/impl/base_function_node.py b/apps/application/flow/step_node/function_node/impl/base_function_node.py
index 4a5c75c8132..f6127e55550 100644
--- a/apps/application/flow/step_node/function_node/impl/base_function_node.py
+++ b/apps/application/flow/step_node/function_node/impl/base_function_node.py
@@ -32,6 +32,8 @@ def write_context(step_variable: Dict, global_variable: Dict, node, workflow):
def valid_reference_value(_type, value, name):
+ if value is None:
+ return
if _type == 'int':
instance_type = int | float
elif _type == 'float':
@@ -49,13 +51,20 @@ def valid_reference_value(_type, value, name):
def convert_value(name: str, value, _type, is_required, source, node):
- if not is_required and value is None:
+ if not is_required and (value is None or (isinstance(value, str) and len(value) == 0)):
return None
if source == 'reference':
+ if value and isinstance(value, list) and len(value) == 0:
+ if not is_required:
+ return None
+ else:
+ raise Exception(f"字段:{name}类型:{_type}值:{value}必填参数")
value = node.workflow_manage.get_reference_field(
value[0],
value[1:])
valid_reference_value(_type, value, name)
+ if value is None:
+ return None
if _type == 'int':
return int(value)
if _type == 'float':
@@ -84,7 +93,8 @@ def convert_value(name: str, value, _type, is_required, source, node):
class BaseFunctionNodeNode(IFunctionNode):
def save_context(self, details, workflow_manage):
self.context['result'] = details.get('result')
- self.answer_text = str(details.get('result'))
+ if self.node_params.get('is_result', False):
+ self.answer_text = str(details.get('result'))
def execute(self, input_field_list, code, **kwargs) -> NodeResult:
params = {field.get('name'): convert_value(field.get('name'), field.get('value'), field.get('type'),
diff --git a/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py b/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py
index d5cc2c5a211..16423eafd61 100644
--- a/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py
+++ b/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py
@@ -16,7 +16,8 @@ class BaseImageGenerateNode(IImageGenerateNode):
def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
self.context['question'] = details.get('question')
- self.answer_text = details.get('answer')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, chat_id,
model_params_setting,
@@ -24,7 +25,8 @@ def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_t
**kwargs) -> NodeResult:
print(model_params_setting)
application = self.workflow_manage.work_flow_post_handler.chat_info.application
- tti_model = get_model_instance_by_model_user_id(model_id, self.flow_params_serializer.data.get('user_id'), **model_params_setting)
+ tti_model = get_model_instance_by_model_user_id(model_id, self.flow_params_serializer.data.get('user_id'),
+ **model_params_setting)
history_message = self.get_history_message(history_chat_record, dialogue_number)
self.context['history_message'] = history_message
question = self.generate_prompt_question(prompt)
diff --git a/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py b/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py
index 3b96f15cd6f..0b405619dde 100644
--- a/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py
+++ b/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py
@@ -62,14 +62,15 @@ def file_id_to_base64(file_id: str):
file = QuerySet(File).filter(id=file_id).first()
file_bytes = file.get_byte()
base64_image = base64.b64encode(file_bytes).decode("utf-8")
- return [base64_image, what(None, file_bytes.tobytes())]
+ return [base64_image, what(None, file_bytes)]
class BaseImageUnderstandNode(IImageUnderstandNode):
def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
self.context['question'] = details.get('question')
- self.answer_text = details.get('answer')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, model_id, system, prompt, dialogue_number, dialogue_type, history_chat_record, stream, chat_id,
model_params_setting,
@@ -171,7 +172,7 @@ def generate_message_list(self, image_model, system: str, prompt: str, history_m
file = QuerySet(File).filter(id=file_id).first()
image_bytes = file.get_byte()
base64_image = base64.b64encode(image_bytes).decode("utf-8")
- image_format = what(None, image_bytes.tobytes())
+ image_format = what(None, image_bytes)
images.append({'type': 'image_url', 'image_url': {'url': f'data:image/{image_format};base64,{base64_image}'}})
messages = [HumanMessage(
content=[
diff --git a/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py b/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py
index 6c9fe97fc69..d5197e9ad11 100644
--- a/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py
+++ b/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py
@@ -14,7 +14,6 @@ def save_context(self, details, workflow_manage):
self.context['result'] = details.get('result')
self.context['tool_params'] = details.get('tool_params')
self.context['mcp_tool'] = details.get('mcp_tool')
- self.answer_text = details.get('result')
def execute(self, mcp_servers, mcp_server, mcp_tool, tool_params, **kwargs) -> NodeResult:
servers = json.loads(mcp_servers)
@@ -27,7 +26,8 @@ async def call_tool(s, session, t, a):
return s
res = asyncio.run(call_tool(servers, mcp_server, mcp_tool, params))
- return NodeResult({'result': [content.text for content in res.content], 'tool_params': params, 'mcp_tool': mcp_tool}, {})
+ return NodeResult(
+ {'result': [content.text for content in res.content], 'tool_params': params, 'mcp_tool': mcp_tool}, {})
def handle_variables(self, tool_params):
# 处理参数中的变量
diff --git a/apps/application/flow/step_node/question_node/impl/base_question_node.py b/apps/application/flow/step_node/question_node/impl/base_question_node.py
index 48a2639b782..e1fd5b86069 100644
--- a/apps/application/flow/step_node/question_node/impl/base_question_node.py
+++ b/apps/application/flow/step_node/question_node/impl/base_question_node.py
@@ -80,7 +80,8 @@ def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
self.context['message_tokens'] = details.get('message_tokens')
self.context['answer_tokens'] = details.get('answer_tokens')
- self.answer_text = details.get('answer')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, chat_record_id,
model_params_setting=None,
diff --git a/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py b/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py
index c85588cd4d2..8f48823f00c 100644
--- a/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py
+++ b/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py
@@ -18,7 +18,9 @@ class BaseSpeechToTextNode(ISpeechToTextNode):
def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
- self.answer_text = details.get('answer')
+ self.context['result'] = details.get('answer')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, stt_model_id, chat_id, audio, **kwargs) -> NodeResult:
stt_model = get_model_instance_by_model_user_id(stt_model_id, self.flow_params_serializer.data.get('user_id'))
@@ -30,7 +32,7 @@ def process_audio_item(audio_item, model):
# 根据file_name 吧文件转成mp3格式
file_format = file.file_name.split('.')[-1]
with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{file_format}') as temp_file:
- temp_file.write(file.get_byte().tobytes())
+ temp_file.write(file.get_byte())
temp_file_path = temp_file.name
with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as temp_amr_file:
temp_mp3_path = temp_amr_file.name
diff --git a/apps/application/flow/step_node/start_node/impl/base_start_node.py b/apps/application/flow/step_node/start_node/impl/base_start_node.py
index bf5203274eb..24b9684714e 100644
--- a/apps/application/flow/step_node/start_node/impl/base_start_node.py
+++ b/apps/application/flow/step_node/start_node/impl/base_start_node.py
@@ -40,10 +40,13 @@ def save_context(self, details, workflow_manage):
self.context['document'] = details.get('document_list')
self.context['image'] = details.get('image_list')
self.context['audio'] = details.get('audio_list')
+ self.context['other'] = details.get('other_list')
self.status = details.get('status')
self.err_message = details.get('err_message')
for key, value in workflow_variable.items():
workflow_manage.context[key] = value
+ for item in details.get('global_fields', []):
+ workflow_manage.context[item.get('key')] = item.get('value')
def get_node_params_serializer_class(self) -> Type[serializers.Serializer]:
pass
@@ -59,7 +62,8 @@ def execute(self, question, **kwargs) -> NodeResult:
'question': question,
'image': self.workflow_manage.image_list,
'document': self.workflow_manage.document_list,
- 'audio': self.workflow_manage.audio_list
+ 'audio': self.workflow_manage.audio_list,
+ 'other': self.workflow_manage.other_list,
}
return NodeResult(node_variable, workflow_variable)
@@ -83,5 +87,6 @@ def get_details(self, index: int, **kwargs):
'image_list': self.context.get('image'),
'document_list': self.context.get('document'),
'audio_list': self.context.get('audio'),
+ 'other_list': self.context.get('other'),
'global_fields': global_fields
}
diff --git a/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py b/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py
index 72c4d3be514..330dc5f5804 100644
--- a/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py
+++ b/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py
@@ -37,7 +37,9 @@ def bytes_to_uploaded_file(file_bytes, file_name="generated_audio.mp3"):
class BaseTextToSpeechNode(ITextToSpeechNode):
def save_context(self, details, workflow_manage):
self.context['answer'] = details.get('answer')
- self.answer_text = details.get('answer')
+ self.context['result'] = details.get('result')
+ if self.node_params.get('is_result', False):
+ self.answer_text = details.get('answer')
def execute(self, tts_model_id, chat_id,
content, model_params_setting=None,
@@ -72,4 +74,5 @@ def get_details(self, index: int, **kwargs):
'content': self.context.get('content'),
'err_message': self.err_message,
'answer': self.context.get('answer'),
+ 'result': self.context.get('result')
}
diff --git a/apps/application/flow/workflow_manage.py b/apps/application/flow/workflow_manage.py
index be91f69be9e..554b0b75f47 100644
--- a/apps/application/flow/workflow_manage.py
+++ b/apps/application/flow/workflow_manage.py
@@ -14,7 +14,7 @@
from functools import reduce
from typing import List, Dict
-from django.db import close_old_connections
+from django.db import close_old_connections, connection
from django.db.models import QuerySet
from django.utils import translation
from django.utils.translation import get_language
@@ -238,6 +238,7 @@ def __init__(self, flow: Flow, params, work_flow_post_handler: WorkFlowPostHandl
base_to_response: BaseToResponse = SystemToResponse(), form_data=None, image_list=None,
document_list=None,
audio_list=None,
+ other_list=None,
start_node_id=None,
start_node_data=None, chat_record=None, child_node=None):
if form_data is None:
@@ -248,12 +249,15 @@ def __init__(self, flow: Flow, params, work_flow_post_handler: WorkFlowPostHandl
document_list = []
if audio_list is None:
audio_list = []
+ if other_list is None:
+ other_list = []
self.start_node_id = start_node_id
self.start_node = None
self.form_data = form_data
self.image_list = image_list
self.document_list = document_list
self.audio_list = audio_list
+ self.other_list = other_list
self.params = params
self.flow = flow
self.context = {}
@@ -294,8 +298,8 @@ def init_fields(self):
if global_fields is not None:
for global_field in global_fields:
global_field_list.append({**global_field, 'node_id': node_id, 'node_name': node_name})
- field_list.sort(key=lambda f: len(f.get('node_name')), reverse=True)
- global_field_list.sort(key=lambda f: len(f.get('node_name')), reverse=True)
+ field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True)
+ global_field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True)
self.field_list = field_list
self.global_field_list = global_field_list
@@ -565,6 +569,8 @@ def hand_event_node_result(self, current_node, node_result_future):
return None
finally:
current_node.node_chunk.end()
+ # 归还链接
+ connection.close()
def run_node_async(self, node):
future = executor.submit(self.run_node, node)
@@ -674,10 +680,16 @@ def get_next_node(self):
return None
@staticmethod
- def dependent_node(up_node_id, node):
+ def dependent_node(edge, node):
+ up_node_id = edge.sourceNodeId
if not node.node_chunk.is_end():
return False
if node.id == up_node_id:
+ if node.context.get('branch_id', None):
+ if edge.sourceAnchorId == f"{node.id}_{node.context.get('branch_id', None)}_right":
+ return True
+ else:
+ return False
if node.type == 'form-node':
if node.context.get('form_data', None) is not None:
return True
@@ -690,9 +702,11 @@ def dependent_node_been_executed(self, node_id):
@param node_id: 需要判断的节点id
@return:
"""
- up_node_id_list = [edge.sourceNodeId for edge in self.flow.edges if edge.targetNodeId == node_id]
- return all([any([self.dependent_node(up_node_id, node) for node in self.node_context]) for up_node_id in
- up_node_id_list])
+ up_edge_list = [edge for edge in self.flow.edges if edge.targetNodeId == node_id]
+ return all(
+ [any([self.dependent_node(edge, node) for node in self.node_context if node.id == edge.sourceNodeId]) for
+ edge in
+ up_edge_list])
def get_up_node_id_list(self, node_id):
up_node_id_list = [edge.sourceNodeId for edge in self.flow.edges if edge.targetNodeId == node_id]
@@ -751,7 +765,10 @@ def get_reference_field(self, node_id: str, fields: List[str]):
if node_id == 'global':
return INode.get_field(self.context, fields)
else:
- return self.get_node_by_id(node_id).get_reference_field(fields)
+ node = self.get_node_by_id(node_id)
+ if node:
+ return node.get_reference_field(fields)
+ return None
def get_workflow_content(self):
context = {
diff --git a/apps/application/migrations/0015_re_database_index.py b/apps/application/migrations/0015_re_database_index.py
index 740a2a2d241..cafe14e209c 100644
--- a/apps/application/migrations/0015_re_database_index.py
+++ b/apps/application/migrations/0015_re_database_index.py
@@ -1,9 +1,8 @@
# Generated by Django 4.2.15 on 2024-09-18 16:14
import logging
-import psycopg2
+import psycopg
from django.db import migrations
-from psycopg2 import extensions
from smartdoc.const import CONFIG
@@ -17,7 +16,7 @@ def get_connect(db_name):
"port": CONFIG.get('DB_PORT')
}
# 建立连接
- connect = psycopg2.connect(**conn_params)
+ connect = psycopg.connect(**conn_params)
return connect
@@ -28,7 +27,7 @@ def sql_execute(conn, reindex_sql: str, alter_database_sql: str):
@param conn:
@param alter_database_sql:
"""
- conn.set_isolation_level(extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+ conn.autocommit = True
with conn.cursor() as cursor:
cursor.execute(reindex_sql, [])
cursor.execute(alter_database_sql, [])
diff --git a/apps/application/models/application.py b/apps/application/models/application.py
index dfe9534e82b..0032271a70b 100644
--- a/apps/application/models/application.py
+++ b/apps/application/models/application.py
@@ -11,7 +11,7 @@
from django.contrib.postgres.fields import ArrayField
from django.db import models
from langchain.schema import HumanMessage, AIMessage
-
+from django.utils.translation import gettext as _
from common.encoder.encoder import SystemEncoder
from common.mixins.app_model_mixin import AppModelMixin
from dataset.models.data_set import DataSet
@@ -167,7 +167,11 @@ def get_human_message(self):
return HumanMessage(content=self.problem_text)
def get_ai_message(self):
- return AIMessage(content=self.answer_text)
+ answer_text = self.answer_text
+ if answer_text is None or len(str(answer_text).strip()) == 0:
+ answer_text = _(
+ 'Sorry, no relevant content was found. Please re-describe your problem or provide more information. ')
+ return AIMessage(content=answer_text)
def get_node_details_runtime_node_id(self, runtime_node_id):
return self.details.get(runtime_node_id, None)
diff --git a/apps/application/serializers/application_serializers.py b/apps/application/serializers/application_serializers.py
index 3792076be7c..b898100160a 100644
--- a/apps/application/serializers/application_serializers.py
+++ b/apps/application/serializers/application_serializers.py
@@ -16,6 +16,7 @@
import uuid
from functools import reduce
from typing import Dict, List
+
from django.contrib.postgres.fields import ArrayField
from django.core import cache, validators
from django.core import signing
@@ -24,8 +25,8 @@
from django.db.models.expressions import RawSQL
from django.http import HttpResponse
from django.template import Template, Context
+from django.utils.translation import gettext_lazy as _, get_language, to_locale
from langchain_mcp_adapters.client import MultiServerMCPClient
-from mcp.client.sse import sse_client
from rest_framework import serializers, status
from rest_framework.utils.formatting import lazy_format
@@ -38,7 +39,7 @@
from common.constants.authentication_type import AuthenticationType
from common.db.search import get_dynamics_model, native_search, native_page_search
from common.db.sql_execute import select_list
-from common.exception.app_exception import AppApiException, NotFound404, AppUnauthorizedFailed, ChatException
+from common.exception.app_exception import AppApiException, NotFound404, AppUnauthorizedFailed
from common.field.common import UploadedImageField, UploadedFileField
from common.models.db_model_manage import DBModelManage
from common.response import result
@@ -57,7 +58,6 @@
from setting.serializers.provider_serializers import ModelSerializer
from smartdoc.conf import PROJECT_DIR
from users.models import User
-from django.utils.translation import gettext_lazy as _, get_language, to_locale
chat_cache = cache.caches['chat_cache']
@@ -148,10 +148,12 @@ class ModelSettingSerializer(serializers.Serializer):
error_messages=ErrMessage.char(_("Thinking process switch")))
reasoning_content_start = serializers.CharField(required=False, allow_null=True, default="",
allow_blank=True, max_length=256,
+ trim_whitespace=False,
error_messages=ErrMessage.char(
_("The thinking process begins to mark")))
reasoning_content_end = serializers.CharField(required=False, allow_null=True, allow_blank=True, default=" ",
max_length=256,
+ trim_whitespace=False,
error_messages=ErrMessage.char(_("End of thinking process marker")))
@@ -162,7 +164,7 @@ class ApplicationWorkflowSerializer(serializers.Serializer):
max_length=256, min_length=1,
error_messages=ErrMessage.char(_("Application Description")))
work_flow = serializers.DictField(required=False, error_messages=ErrMessage.dict(_("Workflow Objects")))
- prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=4096,
+ prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
error_messages=ErrMessage.char(_("Opening remarks")))
@staticmethod
@@ -225,7 +227,7 @@ class ApplicationSerializer(serializers.Serializer):
min_value=0,
max_value=1024,
error_messages=ErrMessage.integer(_("Historical chat records")))
- prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=4096,
+ prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
error_messages=ErrMessage.char(_("Opening remarks")))
dataset_id_list = serializers.ListSerializer(required=False, child=serializers.UUIDField(required=True),
allow_null=True,
@@ -320,6 +322,7 @@ def get_embed(self, with_valid=True, params=None):
def get_query_api_input(self, application, params):
query = ''
+ is_asker = False
if application.work_flow is not None:
work_flow = application.work_flow
if work_flow is not None:
@@ -331,8 +334,10 @@ def get_query_api_input(self, application, params):
if input_field_list is not None:
for field in input_field_list:
if field['assignment_method'] == 'api_input' and field['variable'] in params:
+ if field['variable'] == 'asker':
+ is_asker = True
query += f"&{field['variable']}={params[field['variable']]}"
- if 'asker' in params:
+ if 'asker' in params and not is_asker:
query += f"&asker={params.get('asker')}"
return query
@@ -493,7 +498,7 @@ class Edit(serializers.Serializer):
min_value=0,
max_value=1024,
error_messages=ErrMessage.integer(_("Historical chat records")))
- prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=4096,
+ prologue = serializers.CharField(required=False, allow_null=True, allow_blank=True, max_length=102400,
error_messages=ErrMessage.char(_("Opening remarks")))
dataset_id_list = serializers.ListSerializer(required=False, child=serializers.UUIDField(required=True),
error_messages=ErrMessage.list(_("Related Knowledge Base"))
@@ -1010,7 +1015,8 @@ def profile(self, with_valid=True):
'stt_autosend': application.stt_autosend,
'file_upload_enable': application.file_upload_enable,
'file_upload_setting': application.file_upload_setting,
- 'work_flow': application.work_flow,
+ 'work_flow': {'nodes': [node for node in ((application.work_flow or {}).get('nodes', []) or []) if
+ node.get('id') == 'base-node']},
'show_source': application_access_token.show_source,
'language': application_access_token.language,
**application_setting_dict})
@@ -1071,6 +1077,7 @@ def edit(self, instance: Dict, with_valid=True):
for update_key in update_keys:
if update_key in instance and instance.get(update_key) is not None:
application.__setattr__(update_key, instance.get(update_key))
+ print(application.name)
application.save()
if 'dataset_id_list' in instance:
@@ -1089,6 +1096,7 @@ def edit(self, instance: Dict, with_valid=True):
chat_cache.clear_by_application_id(application_id)
application_access_token = QuerySet(ApplicationAccessToken).filter(application_id=application_id).first()
# 更新缓存数据
+ print(application.name)
get_application_access_token(application_access_token.access_token, False)
return self.one(with_valid=False)
@@ -1141,6 +1149,8 @@ def get_work_flow_model(instance):
instance['file_upload_enable'] = node_data['file_upload_enable']
if 'file_upload_setting' in node_data:
instance['file_upload_setting'] = node_data['file_upload_setting']
+ if 'name' in node_data:
+ instance['name'] = node_data['name']
break
def speech_to_text(self, file, with_valid=True):
@@ -1318,7 +1328,12 @@ class McpServers(serializers.Serializer):
def get_mcp_servers(self, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
+ if '"stdio"' in self.data.get('mcp_servers'):
+ raise AppApiException(500, _('stdio is not supported'))
servers = json.loads(self.data.get('mcp_servers'))
+ for server, config in servers.items():
+ if config.get('transport') not in ['sse', 'streamable_http']:
+ raise AppApiException(500, _('Only support transport=sse or transport=streamable_http'))
async def get_mcp_tools(servers):
async with MultiServerMCPClient(servers) as client:
diff --git a/apps/application/serializers/chat_message_serializers.py b/apps/application/serializers/chat_message_serializers.py
index 2194028e6dd..e0ea7e9f555 100644
--- a/apps/application/serializers/chat_message_serializers.py
+++ b/apps/application/serializers/chat_message_serializers.py
@@ -213,12 +213,21 @@ def get_message(instance):
return instance.get('messages')[-1].get('content')
@staticmethod
- def generate_chat(chat_id, application_id, message, client_id):
+ def generate_chat(chat_id, application_id, message, client_id, asker=None):
if chat_id is None:
chat_id = str(uuid.uuid1())
chat = QuerySet(Chat).filter(id=chat_id).first()
if chat is None:
- Chat(id=chat_id, application_id=application_id, abstract=message[0:1024], client_id=client_id).save()
+ asker_dict = {'user_name': '游客'}
+ if asker is not None:
+ if isinstance(asker, str):
+ asker_dict = {
+ 'user_name': asker
+ }
+ elif isinstance(asker, dict):
+ asker_dict = asker
+ Chat(id=chat_id, application_id=application_id, abstract=message[0:1024], client_id=client_id,
+ asker=asker_dict).save()
return chat_id
def chat(self, instance: Dict, with_valid=True):
@@ -232,7 +241,8 @@ def chat(self, instance: Dict, with_valid=True):
application_id = self.data.get('application_id')
client_id = self.data.get('client_id')
client_type = self.data.get('client_type')
- chat_id = self.generate_chat(chat_id, application_id, message, client_id)
+ chat_id = self.generate_chat(chat_id, application_id, message, client_id,
+ asker=instance.get('form_data', {}).get("asker"))
return ChatMessageSerializer(
data={
'chat_id': chat_id, 'message': message,
@@ -245,6 +255,7 @@ def chat(self, instance: Dict, with_valid=True):
'image_list': instance.get('image_list', []),
'document_list': instance.get('document_list', []),
'audio_list': instance.get('audio_list', []),
+ 'other_list': instance.get('other_list', []),
}
).chat(base_to_response=OpenaiToResponse())
@@ -274,6 +285,7 @@ class ChatMessageSerializer(serializers.Serializer):
image_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("picture")))
document_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("document")))
audio_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Audio")))
+ other_list = serializers.ListField(required=False, error_messages=ErrMessage.list(_("Other")))
child_node = serializers.DictField(required=False, allow_null=True,
error_messages=ErrMessage.dict(_("Child Nodes")))
@@ -372,6 +384,7 @@ def chat_work_flow(self, chat_info: ChatInfo, base_to_response):
image_list = self.data.get('image_list')
document_list = self.data.get('document_list')
audio_list = self.data.get('audio_list')
+ other_list = self.data.get('other_list')
user_id = chat_info.application.user_id
chat_record_id = self.data.get('chat_record_id')
chat_record = None
@@ -382,13 +395,14 @@ def chat_work_flow(self, chat_info: ChatInfo, base_to_response):
work_flow_manage = WorkflowManage(Flow.new_instance(chat_info.work_flow_version.work_flow),
{'history_chat_record': history_chat_record, 'question': message,
'chat_id': chat_info.chat_id, 'chat_record_id': str(
- uuid.uuid1()) if chat_record is None else chat_record.id,
+ uuid.uuid1()) if chat_record is None else str(chat_record.id),
'stream': stream,
're_chat': re_chat,
'client_id': client_id,
'client_type': client_type,
'user_id': user_id}, WorkFlowPostHandler(chat_info, client_id, client_type),
base_to_response, form_data, image_list, document_list, audio_list,
+ other_list,
self.data.get('runtime_node_id'),
self.data.get('node_data'), chat_record, self.data.get('child_node'))
r = work_flow_manage.run()
diff --git a/apps/application/serializers/chat_serializers.py b/apps/application/serializers/chat_serializers.py
index b90194d5ae2..bc397fecf4a 100644
--- a/apps/application/serializers/chat_serializers.py
+++ b/apps/application/serializers/chat_serializers.py
@@ -13,8 +13,9 @@
from functools import reduce
from io import BytesIO
from typing import Dict
-import pytz
+
import openpyxl
+import pytz
from django.core import validators
from django.core.cache import caches
from django.db import transaction, models
@@ -33,8 +34,8 @@
ModelSettingSerializer
from application.serializers.chat_message_serializers import ChatInfo
from common.constants.permission_constants import RoleConstants
-from common.db.search import native_search, native_page_search, page_search, get_dynamics_model
-from common.exception.app_exception import AppApiException
+from common.db.search import native_search, native_page_search, page_search, get_dynamics_model, native_page_handler
+from common.exception.app_exception import AppApiException, AppUnauthorizedFailed
from common.util.common import post
from common.util.field_message import ErrMessage
from common.util.file_util import get_file_content
@@ -144,7 +145,8 @@ def get_query_set(self, select_ids=None):
'trample_num': models.IntegerField(),
'comparer': models.CharField(),
'application_chat.update_time': models.DateTimeField(),
- 'application_chat.id': models.UUIDField(), }))
+ 'application_chat.id': models.UUIDField(),
+ 'application_chat_record_temp.id': models.UUIDField()}))
base_query_dict = {'application_chat.application_id': self.data.get("application_id"),
'application_chat.update_time__gte': start_time,
@@ -174,7 +176,14 @@ def get_query_set(self, select_ids=None):
condition = base_condition & min_trample_query
else:
condition = base_condition
- return query_set.filter(condition).order_by("-application_chat.update_time")
+ inner_queryset = QuerySet(Chat).filter(application_id=self.data.get("application_id"))
+ if 'abstract' in self.data and self.data.get('abstract') is not None:
+ inner_queryset = inner_queryset.filter(abstract__icontains=self.data.get('abstract'))
+
+ return {
+ 'inner_queryset': inner_queryset,
+ 'default_queryset': query_set.filter(condition).order_by("-application_chat.update_time")
+ }
def list(self, with_valid=True):
if with_valid:
@@ -215,7 +224,8 @@ def to_row(row: Dict):
reference_paragraph,
"\n".join([
f"{improve_paragraph_list[index].get('title')}\n{improve_paragraph_list[index].get('content')}"
- for index in range(len(improve_paragraph_list))]),
+ for index in range(len(improve_paragraph_list))
+ ]) if improve_paragraph_list is not None else "",
row.get('asker').get('user_name'),
row.get('message_tokens') + row.get('answer_tokens'), row.get('run_time'),
str(row.get('create_time').astimezone(pytz.timezone(TIME_ZONE)).strftime('%Y-%m-%d %H:%M:%S')
@@ -225,55 +235,90 @@ def export(self, data, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
- data_list = native_search(self.get_query_set(data.get('select_ids')),
- select_string=get_file_content(
- os.path.join(PROJECT_DIR, "apps", "application", 'sql',
- 'export_application_chat.sql')),
- with_table_name=False)
+ batch_size = 2000
- batch_size = 500
+ select_sql = get_file_content(
+ os.path.join(
+ PROJECT_DIR,
+ "apps",
+ "application",
+ "sql",
+ "export_application_chat.sql"
+ )
+ )
def stream_response():
- workbook = openpyxl.Workbook()
- worksheet = workbook.active
- worksheet.title = 'Sheet1'
-
- headers = [gettext('Conversation ID'), gettext('summary'), gettext('User Questions'),
- gettext('Problem after optimization'),
- gettext('answer'), gettext('User feedback'),
- gettext('Reference segment number'),
- gettext('Section title + content'),
- gettext('Annotation'), gettext('USER'), gettext('Consuming tokens'),
- gettext('Time consumed (s)'),
- gettext('Question Time')]
- for col_idx, header in enumerate(headers, 1):
- cell = worksheet.cell(row=1, column=col_idx)
- cell.value = header
-
- for i in range(0, len(data_list), batch_size):
- batch_data = data_list[i:i + batch_size]
-
- for row_idx, row in enumerate(batch_data, start=i + 2):
- for col_idx, value in enumerate(self.to_row(row), 1):
- cell = worksheet.cell(row=row_idx, column=col_idx)
- if isinstance(value, str):
- value = re.sub(ILLEGAL_CHARACTERS_RE, '', value)
- if isinstance(value, datetime.datetime):
- eastern = pytz.timezone(TIME_ZONE)
- c = datetime.timezone(eastern._utcoffset)
- value = value.astimezone(c)
- cell.value = value
-
- output = BytesIO()
- workbook.save(output)
- output.seek(0)
- yield output.getvalue()
- output.close()
- workbook.close()
-
- response = StreamingHttpResponse(stream_response(),
- content_type='application/vnd.open.xmlformats-officedocument.spreadsheetml.sheet')
+ import tempfile
+
+ headers = [
+ gettext('Conversation ID'),
+ gettext('summary'),
+ gettext('User Questions'),
+ gettext('Problem after optimization'),
+ gettext('answer'),
+ gettext('User feedback'),
+ gettext('Reference segment number'),
+ gettext('Section title + content'),
+ gettext('Annotation'),
+ gettext('USER'),
+ gettext('Consuming tokens'),
+ gettext('Time consumed (s)'),
+ gettext('Question Time')
+ ]
+
+ with tempfile.NamedTemporaryFile(suffix=".xlsx") as tmp:
+
+ workbook = openpyxl.Workbook(write_only=True)
+ worksheet = workbook.create_sheet(title="Sheet1")
+
+ # 写表头
+ worksheet.append(headers)
+
+ for data_list in native_page_handler(
+ batch_size,
+ self.get_query_set(data.get('select_ids')),
+ primary_key='application_chat_record_temp.id',
+ primary_queryset='default_queryset',
+ get_primary_value=lambda item: item.get('id'),
+ select_string=select_sql,
+ with_table_name=False
+ ):
+
+ for row in data_list:
+
+ row_values = []
+ for value in self.to_row(row):
+
+ if isinstance(value, str):
+ value = re.sub(ILLEGAL_CHARACTERS_RE, '', value)
+
+ elif isinstance(value, datetime.datetime):
+ eastern = pytz.timezone(TIME_ZONE)
+ c = datetime.timezone(eastern._utcoffset)
+ value = value.astimezone(c)
+
+ row_values.append(value)
+
+ worksheet.append(row_values)
+
+ workbook.save(tmp.name)
+ workbook.close()
+
+ # 分块返回文件
+ with open(tmp.name, "rb") as f:
+ while True:
+ chunk = f.read(8192)
+ if not chunk:
+ break
+ yield chunk
+
+ response = StreamingHttpResponse(
+ stream_response(),
+ content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+ )
+
response['Content-Disposition'] = 'attachment; filename="data.xlsx"'
+
return response
def page(self, current_page: int, page_size: int, with_valid=True):
@@ -476,6 +521,13 @@ class Query(serializers.Serializer):
chat_id = serializers.UUIDField(required=True)
order_asc = serializers.BooleanField(required=False, allow_null=True)
+ def is_valid(self, *, raise_exception=False):
+ super().is_valid(raise_exception=True)
+ exist = QuerySet(Chat).filter(id=self.data.get("chat_id"),
+ application_id=self.data.get("application_id")).exists()
+ if not exist:
+ raise AppUnauthorizedFailed(403, _('No permission to access'))
+
def list(self, with_valid=True):
if with_valid:
self.is_valid(raise_exception=True)
diff --git a/apps/application/sql/export_application_chat.sql b/apps/application/sql/export_application_chat.sql
index bb265ea5b02..99c8f7a3172 100644
--- a/apps/application/sql/export_application_chat.sql
+++ b/apps/application/sql/export_application_chat.sql
@@ -1,38 +1,39 @@
-SELECT
- application_chat."id" as chat_id,
- application_chat.abstract as abstract,
- application_chat_record_temp.problem_text as problem_text,
- application_chat_record_temp.answer_text as answer_text,
- application_chat_record_temp.message_tokens as message_tokens,
- application_chat_record_temp.answer_tokens as answer_tokens,
- application_chat_record_temp.run_time as run_time,
- application_chat_record_temp.details::JSON as details,
- application_chat_record_temp."index" as "index",
- application_chat_record_temp.improve_paragraph_list as improve_paragraph_list,
- application_chat_record_temp.vote_status as vote_status,
- application_chat_record_temp.create_time as create_time,
- to_json(application_chat.asker) as asker
-FROM
- application_chat application_chat
- LEFT JOIN (
- SELECT COUNT
- ( "id" ) AS chat_record_count,
- SUM ( CASE WHEN "vote_status" = '0' THEN 1 ELSE 0 END ) AS star_num,
- SUM ( CASE WHEN "vote_status" = '1' THEN 1 ELSE 0 END ) AS trample_num,
- SUM ( CASE WHEN array_length( application_chat_record.improve_paragraph_id_list, 1 ) IS NULL THEN 0 ELSE array_length( application_chat_record.improve_paragraph_id_list, 1 ) END ) AS mark_sum,
- chat_id
- FROM
- application_chat_record
- GROUP BY
- application_chat_record.chat_id
- ) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
- LEFT JOIN (
- SELECT
- *,
- CASE
- WHEN array_length( application_chat_record.improve_paragraph_id_list, 1 ) IS NULL THEN
- '{}' ELSE ( SELECT ARRAY_AGG ( row_to_json ( paragraph ) ) FROM paragraph WHERE "id" = ANY ( application_chat_record.improve_paragraph_id_list ) )
- END as improve_paragraph_list
- FROM
- application_chat_record application_chat_record
- ) application_chat_record_temp ON application_chat_record_temp.chat_id = application_chat."id"
\ No newline at end of file
+SELECT application_chat_record_temp.id AS id,
+ application_chat."id" as chat_id,
+ application_chat.abstract as abstract,
+ application_chat_record_temp.problem_text as problem_text,
+ application_chat_record_temp.answer_text as answer_text,
+ application_chat_record_temp.message_tokens as message_tokens,
+ application_chat_record_temp.answer_tokens as answer_tokens,
+ application_chat_record_temp.run_time as run_time,
+ application_chat_record_temp.details::JSON as details, application_chat_record_temp."index" as "index",
+ application_chat_record_temp.improve_paragraph_list as improve_paragraph_list,
+ application_chat_record_temp.vote_status as vote_status,
+ application_chat_record_temp.create_time as create_time,
+ to_json(application_chat.asker) as asker
+FROM application_chat application_chat
+ LEFT JOIN (SELECT COUNT
+ ("id") AS chat_record_count,
+ SUM(CASE WHEN "vote_status" = '0' THEN 1 ELSE 0 END) AS star_num,
+ SUM(CASE WHEN "vote_status" = '1' THEN 1 ELSE 0 END) AS trample_num,
+ SUM(CASE
+ WHEN array_length(application_chat_record.improve_paragraph_id_list, 1) IS NULL
+ THEN 0
+ ELSE array_length(application_chat_record.improve_paragraph_id_list, 1) END) AS mark_sum,
+ chat_id
+ FROM application_chat_record
+ WHERE chat_id IN (SELECT id
+ FROM application_chat ${inner_queryset})
+ GROUP BY application_chat_record.chat_id) chat_record_temp
+ ON application_chat."id" = chat_record_temp.chat_id
+ LEFT JOIN (SELECT *,
+ CASE
+ WHEN array_length(application_chat_record.improve_paragraph_id_list, 1) IS NULL THEN
+ '{}'
+ ELSE (SELECT ARRAY_AGG(row_to_json(paragraph))
+ FROM paragraph
+ WHERE "id" = ANY (application_chat_record.improve_paragraph_id_list))
+ END as improve_paragraph_list
+ FROM application_chat_record application_chat_record) application_chat_record_temp
+ ON application_chat_record_temp.chat_id = application_chat."id"
+ ${default_queryset}
\ No newline at end of file
diff --git a/apps/application/sql/list_application_chat.sql b/apps/application/sql/list_application_chat.sql
index 7f3e1680c99..c9f83c6b7c3 100644
--- a/apps/application/sql/list_application_chat.sql
+++ b/apps/application/sql/list_application_chat.sql
@@ -11,6 +11,9 @@ FROM
chat_id
FROM
application_chat_record
+ WHERE chat_id IN (
+ SELECT id FROM application_chat ${inner_queryset})
GROUP BY
application_chat_record.chat_id
- ) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
\ No newline at end of file
+ ) chat_record_temp ON application_chat."id" = chat_record_temp.chat_id
+${default_queryset}
\ No newline at end of file
diff --git a/apps/application/swagger_api/application_api.py b/apps/application/swagger_api/application_api.py
index 2c9cbd86bf4..024279832b1 100644
--- a/apps/application/swagger_api/application_api.py
+++ b/apps/application/swagger_api/application_api.py
@@ -61,8 +61,6 @@ def get_response_body_api():
'user_id': openapi.Schema(type=openapi.TYPE_STRING, title=_("Affiliation user"),
description=_("Affiliation user")),
- 'status': openapi.Schema(type=openapi.TYPE_BOOLEAN, title=_("Is publish"), description=_('Is publish')),
-
'create_time': openapi.Schema(type=openapi.TYPE_STRING, title=_("Creation time"),
description=_('Creation time')),
@@ -302,7 +300,19 @@ def get_request_body_api():
'no_references_prompt': openapi.Schema(type=openapi.TYPE_STRING,
title=_("No citation segmentation prompt"),
default="{question}",
- description=_("No citation segmentation prompt"))
+ description=_("No citation segmentation prompt")),
+ 'reasoning_content_enable': openapi.Schema(type=openapi.TYPE_BOOLEAN,
+ title=_("Reasoning enable"),
+ default=False,
+ description=_("Reasoning enable")),
+ 'reasoning_content_end': openapi.Schema(type=openapi.TYPE_STRING,
+ title=_("Reasoning end tag"),
+ default="",
+ description=_("Reasoning end tag")),
+ "reasoning_content_start": openapi.Schema(type=openapi.TYPE_STRING,
+ title=_("Reasoning start tag"),
+ default="",
+ description=_("Reasoning start tag"))
}
)
diff --git a/apps/application/swagger_api/chat_api.py b/apps/application/swagger_api/chat_api.py
index 54b5678f747..f27a19c200e 100644
--- a/apps/application/swagger_api/chat_api.py
+++ b/apps/application/swagger_api/chat_api.py
@@ -326,11 +326,6 @@ def get_request_params_api():
type=openapi.TYPE_STRING,
required=True,
description=_('Application ID')),
- openapi.Parameter(name='history_day',
- in_=openapi.IN_QUERY,
- type=openapi.TYPE_NUMBER,
- required=True,
- description=_('Historical days')),
openapi.Parameter(name='abstract', in_=openapi.IN_QUERY, type=openapi.TYPE_STRING, required=False,
description=_("abstract")),
openapi.Parameter(name='min_star', in_=openapi.IN_QUERY, type=openapi.TYPE_INTEGER, required=False,
diff --git a/apps/application/views/application_version_views.py b/apps/application/views/application_version_views.py
index de900936268..1cd42a643a0 100644
--- a/apps/application/views/application_version_views.py
+++ b/apps/application/views/application_version_views.py
@@ -48,7 +48,11 @@ class Page(APIView):
ApplicationVersionApi.Query.get_request_params_api()),
responses=result.get_page_api_response(ApplicationVersionApi.get_response_body_api()),
tags=[_('Application/Version')])
- @has_permissions(PermissionConstants.APPLICATION_READ, compare=CompareConstants.AND)
+ @has_permissions(PermissionConstants.APPLICATION_READ,
+ ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
+ [lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND), compare=CompareConstants.AND)
def get(self, request: Request, application_id: str, current_page: int, page_size: int):
return result.success(
ApplicationVersionSerializer.Query(
@@ -65,7 +69,14 @@ class Operate(APIView):
manual_parameters=ApplicationVersionApi.Operate.get_request_params_api(),
responses=result.get_api_response(ApplicationVersionApi.get_response_body_api()),
tags=[_('Application/Version')])
- @has_permissions(PermissionConstants.APPLICATION_READ, compare=CompareConstants.AND)
+ @has_permissions(PermissionConstants.APPLICATION_READ, ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
+ [lambda r, keywords: Permission(
+ group=Group.APPLICATION,
+ operate=Operate.USE,
+ dynamic_tag=keywords.get(
+ 'application_id'))],
+ compare=CompareConstants.AND),
+ compare=CompareConstants.AND)
def get(self, request: Request, application_id: str, work_flow_version_id: str):
return result.success(
ApplicationVersionSerializer.Operate(
diff --git a/apps/application/views/application_views.py b/apps/application/views/application_views.py
index f16041d1de3..8c3e8059bcb 100644
--- a/apps/application/views/application_views.py
+++ b/apps/application/views/application_views.py
@@ -7,16 +7,6 @@
@desc:
"""
-from django.core import cache
-from django.http import HttpResponse
-from django.utils.translation import gettext_lazy as _, gettext
-from drf_yasg.utils import swagger_auto_schema
-from langchain_core.prompts import PromptTemplate
-from rest_framework.decorators import action
-from rest_framework.parsers import MultiPartParser
-from rest_framework.request import Request
-from rest_framework.views import APIView
-
from application.serializers.application_serializers import ApplicationSerializer
from application.serializers.application_statistics_serializers import ApplicationStatisticsSerializer
from application.swagger_api.application_api import ApplicationApi
@@ -31,6 +21,14 @@
from common.swagger_api.common_api import CommonApi
from common.util.common import query_params_to_single_dict
from dataset.serializers.dataset_serializers import DataSetSerializers
+from django.core import cache
+from django.http import HttpResponse
+from django.utils.translation import gettext_lazy as _
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework.decorators import action
+from rest_framework.parsers import MultiPartParser
+from rest_framework.request import Request
+from rest_framework.views import APIView
chat_cache = cache.caches['chat_cache']
@@ -494,7 +492,7 @@ def get(self, request: Request):
class HitTest(APIView):
authentication_classes = [TokenAuth]
- @action(methods="GET", detail=False)
+ @action(methods="PUT", detail=False)
@swagger_auto_schema(operation_summary=_("Hit Test List"), operation_id=_("Hit Test List"),
manual_parameters=CommonApi.HitTestApi.get_request_params_api(),
responses=result.get_api_array_response(CommonApi.HitTestApi.get_response_body_api()),
@@ -505,15 +503,15 @@ class HitTest(APIView):
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
dynamic_tag=keywords.get('application_id'))],
compare=CompareConstants.AND))
- def get(self, request: Request, application_id: str):
- return result.success(
- ApplicationSerializer.HitTest(data={'id': application_id, 'user_id': request.user.id,
- "query_text": request.query_params.get("query_text"),
- "top_number": request.query_params.get("top_number"),
- 'similarity': request.query_params.get('similarity'),
- 'search_mode': request.query_params.get(
- 'search_mode')}).hit_test(
- ))
+ def put(self, request: Request, application_id: str):
+ return result.success(ApplicationSerializer.HitTest(data={
+ 'id': application_id,
+ 'user_id': request.user.id,
+ "query_text": request.data.get("query_text"),
+ "top_number": request.data.get("top_number"),
+ 'similarity': request.data.get('similarity'),
+ 'search_mode': request.data.get('search_mode')}
+ ).hit_test())
class Publish(APIView):
authentication_classes = [TokenAuth]
diff --git a/apps/application/views/chat_views.py b/apps/application/views/chat_views.py
index 0415f8208dc..30d54fa65a4 100644
--- a/apps/application/views/chat_views.py
+++ b/apps/application/views/chat_views.py
@@ -59,7 +59,8 @@ class Export(APIView):
@has_permissions(
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
@log(menu='Conversation Log', operate="Export conversation",
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
@@ -144,6 +145,8 @@ def post(self, request: Request, chat_id: str):
'document_list') if 'document_list' in request.data else [],
'audio_list': request.data.get(
'audio_list') if 'audio_list' in request.data else [],
+ 'other_list': request.data.get(
+ 'other_list') if 'other_list' in request.data else [],
'client_type': request.auth.client_type,
'node_id': request.data.get('node_id', None),
'runtime_node_id': request.data.get('runtime_node_id', None),
@@ -162,7 +165,9 @@ def post(self, request: Request, chat_id: str):
@has_permissions(
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND
+ )
)
def get(self, request: Request, application_id: str):
return result.success(ChatSerializers.Query(
@@ -180,8 +185,7 @@ class Operate(APIView):
[RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.MANAGE,
dynamic_tag=keywords.get('application_id'))],
- compare=CompareConstants.AND),
- compare=CompareConstants.AND)
+ compare=CompareConstants.AND))
@log(menu='Conversation Log', operate="Delete a conversation",
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
def delete(self, request: Request, application_id: str, chat_id: str):
@@ -204,7 +208,8 @@ class ClientChatHistoryPage(APIView):
@has_permissions(
ViewPermission([RoleConstants.APPLICATION_ACCESS_TOKEN],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
def get(self, request: Request, application_id: str, current_page: int, page_size: int):
return result.success(ChatSerializers.ClientChatHistory(
@@ -239,7 +244,7 @@ def delete(self, request: Request, application_id: str, chat_id: str):
request_body=ChatClientHistoryApi.Operate.ReAbstract.get_request_body_api(),
tags=[_("Application/Conversation Log")])
@has_permissions(ViewPermission(
- [RoleConstants.APPLICATION_ACCESS_TOKEN],
+ [RoleConstants.APPLICATION_ACCESS_TOKEN, RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
dynamic_tag=keywords.get('application_id'))],
compare=CompareConstants.AND),
@@ -265,7 +270,8 @@ class Page(APIView):
@has_permissions(
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
def get(self, request: Request, application_id: str, current_page: int, page_size: int):
return result.success(ChatSerializers.Query(
@@ -290,7 +296,8 @@ class Operate(APIView):
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY,
RoleConstants.APPLICATION_ACCESS_TOKEN],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
def get(self, request: Request, application_id: str, chat_id: str, chat_record_id: str):
return result.success(ChatRecordSerializer.Operate(
@@ -308,7 +315,8 @@ def get(self, request: Request, application_id: str, chat_id: str, chat_record_i
@has_permissions(
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
def get(self, request: Request, application_id: str, chat_id: str):
return result.success(ChatRecordSerializer.Query(
@@ -327,9 +335,11 @@ class Page(APIView):
tags=[_("Application/Conversation Log")]
)
@has_permissions(
- ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY],
+ ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY,
+ RoleConstants.APPLICATION_ACCESS_TOKEN],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
def get(self, request: Request, application_id: str, chat_id: str, current_page: int, page_size: int):
return result.success(ChatRecordSerializer.Query(
@@ -352,7 +362,8 @@ class Vote(APIView):
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY,
RoleConstants.APPLICATION_ACCESS_TOKEN],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND)
)
@log(menu='Conversation Log', operate="Like, Dislike",
get_operation_object=lambda r, k: get_application_operation_object(k.get('application_id')))
@@ -375,7 +386,7 @@ class ChatRecordImprove(APIView):
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
dynamic_tag=keywords.get('application_id'))]
- ))
+ , compare=CompareConstants.AND))
def get(self, request: Request, application_id: str, chat_id: str, chat_record_id: str):
return result.success(ChatRecordSerializer.ChatRecordImprove(
data={'chat_id': chat_id, 'chat_record_id': chat_record_id}).get())
@@ -395,7 +406,7 @@ class Improve(APIView):
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
dynamic_tag=keywords.get('application_id'))],
-
+ compare=CompareConstants.AND
), ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.DATASET,
operate=Operate.MANAGE,
@@ -422,6 +433,7 @@ def put(self, request: Request, application_id: str, chat_id: str, chat_record_i
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND
), ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.DATASET,
@@ -449,6 +461,7 @@ class Operate(APIView):
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
dynamic_tag=keywords.get('application_id'))],
+ compare=CompareConstants.AND
), ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
[lambda r, keywords: Permission(group=Group.DATASET,
@@ -497,7 +510,8 @@ class UploadFile(APIView):
ViewPermission([RoleConstants.ADMIN, RoleConstants.USER, RoleConstants.APPLICATION_KEY,
RoleConstants.APPLICATION_ACCESS_TOKEN],
[lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
- dynamic_tag=keywords.get('application_id'))])
+ dynamic_tag=keywords.get('application_id'))]
+ , compare=CompareConstants.AND)
)
def post(self, request: Request, application_id: str, chat_id: str):
files = request.FILES.getlist('file')
diff --git a/apps/common/auth/handle/impl/user_token.py b/apps/common/auth/handle/impl/user_token.py
index dbb6bd2b51a..bdb041f9f79 100644
--- a/apps/common/auth/handle/impl/user_token.py
+++ b/apps/common/auth/handle/impl/user_token.py
@@ -6,18 +6,18 @@
@date：2024/3/14 03:02
@desc: 用户认证
"""
+from django.core import cache
from django.db.models import QuerySet
+from django.utils.translation import gettext_lazy as _
from common.auth.handle.auth_base_handle import AuthBaseHandle
from common.constants.authentication_type import AuthenticationType
from common.constants.permission_constants import RoleConstants, get_permission_list_by_role, Auth
from common.exception.app_exception import AppAuthenticationFailed
-from smartdoc.settings import JWT_AUTH
+from smartdoc.const import CONFIG
from users.models import User
-from django.core import cache
-
from users.models.user import get_user_dynamics_permission
-from django.utils.translation import gettext_lazy as _
+
token_cache = cache.caches['token_cache']
@@ -35,7 +35,7 @@ def handle(self, request, token: str, get_token_details):
auth_details = get_token_details()
user = QuerySet(User).get(id=auth_details['id'])
# 续期
- token_cache.touch(token, timeout=JWT_AUTH['JWT_EXPIRATION_DELTA'].total_seconds())
+ token_cache.touch(token, timeout=CONFIG.get_session_timeout())
rule = RoleConstants[user.role]
permission_list = get_permission_list_by_role(RoleConstants[user.role])
# 获取用户的应用和知识库的权限
diff --git a/apps/common/config/embedding_config.py b/apps/common/config/embedding_config.py
index a6e9ab9aa9b..69081be055d 100644
--- a/apps/common/config/embedding_config.py
+++ b/apps/common/config/embedding_config.py
@@ -11,35 +11,50 @@
from common.cache.mem_cache import MemCache
-lock = threading.Lock()
+_lock = threading.Lock()
+locks = {}
class ModelManage:
cache = MemCache('model', {})
up_clear_time = time.time()
+ @staticmethod
+ def _get_lock(_id):
+ lock = locks.get(_id)
+ if lock is None:
+ with _lock:
+ lock = locks.get(_id)
+ if lock is None:
+ lock = threading.Lock()
+ locks[_id] = lock
+
+ return lock
+
@staticmethod
def get_model(_id, get_model):
- # 获取锁
- lock.acquire()
- try:
- model_instance = ModelManage.cache.get(_id)
- if model_instance is None or not model_instance.is_cache_model():
+ model_instance = ModelManage.cache.get(_id)
+ if model_instance is None:
+ lock = ModelManage._get_lock(_id)
+ with lock:
+ model_instance = ModelManage.cache.get(_id)
+ if model_instance is None:
+ model_instance = get_model(_id)
+ ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
+ else:
+ if model_instance.is_cache_model():
+ ModelManage.cache.touch(_id, timeout=60 * 60 * 8)
+ else:
model_instance = get_model(_id)
- ModelManage.cache.set(_id, model_instance, timeout=60 * 30)
- return model_instance
- # 续期
- ModelManage.cache.touch(_id, timeout=60 * 30)
- ModelManage.clear_timeout_cache()
- return model_instance
- finally:
- # 释放锁
- lock.release()
+ ModelManage.cache.set(_id, model_instance, timeout=60 * 60 * 8)
+ ModelManage.clear_timeout_cache()
+ return model_instance
@staticmethod
def clear_timeout_cache():
- if time.time() - ModelManage.up_clear_time > 60:
- ModelManage.cache.clear_timeout_data()
+ if time.time() - ModelManage.up_clear_time > 60 * 60:
+ threading.Thread(target=lambda: ModelManage.cache.clear_timeout_data()).start()
+ ModelManage.up_clear_time = time.time()
@staticmethod
def delete_key(_id):
diff --git a/apps/common/db/search.py b/apps/common/db/search.py
index bef42a1414a..07ecd1b0262 100644
--- a/apps/common/db/search.py
+++ b/apps/common/db/search.py
@@ -170,6 +170,51 @@ def native_page_search(current_page: int, page_size: int, queryset: QuerySet | D
return Page(total.get("count"), list(map(post_records_handler, result)), current_page, page_size)
+def native_page_handler(page_size: int,
+ queryset: QuerySet | Dict[str, QuerySet],
+ select_string: str,
+ field_replace_dict=None,
+ with_table_name=False,
+ primary_key=None,
+ get_primary_value=None,
+ primary_queryset: str = None,
+ ):
+ if isinstance(queryset, Dict):
+ exec_sql, exec_params = generate_sql_by_query_dict({**queryset,
+ primary_queryset: queryset[primary_queryset].order_by(
+ primary_key)}, select_string, field_replace_dict, with_table_name)
+ else:
+ exec_sql, exec_params = generate_sql_by_query(queryset.order_by(
+ primary_key), select_string, field_replace_dict, with_table_name)
+ total_sql = "SELECT \"count\"(*) FROM (%s) temp" % exec_sql
+ total = select_one(total_sql, exec_params)
+ processed_count = 0
+ last_id = None
+ while processed_count < total.get("count"):
+ if last_id is not None:
+ if isinstance(queryset, Dict):
+ exec_sql, exec_params = generate_sql_by_query_dict({**queryset,
+ primary_queryset: queryset[primary_queryset].filter(
+ **{f"{primary_key}__gt": last_id}).order_by(
+ primary_key)},
+ select_string, field_replace_dict,
+ with_table_name)
+ else:
+ exec_sql, exec_params = generate_sql_by_query(
+ queryset.filter(**{f"{primary_key}__gt": last_id}).order_by(
+ primary_key),
+ select_string, field_replace_dict,
+ with_table_name)
+ limit_sql = connections[DEFAULT_DB_ALIAS].ops.limit_offset_sql(
+ 0, page_size
+ )
+ page_sql = exec_sql + " " + limit_sql
+ result = select_list(page_sql, exec_params)
+ yield result
+ processed_count += page_size
+ last_id = get_primary_value(result[-1])
+
+
def get_field_replace_dict(queryset: QuerySet):
"""
获取需要替换的字段 默认 “xxx.xxx”需要被替换成 “xxx”."xxx"
diff --git a/apps/common/event/listener_manage.py b/apps/common/event/listener_manage.py
index 72d16ebb523..6899c31f33e 100644
--- a/apps/common/event/listener_manage.py
+++ b/apps/common/event/listener_manage.py
@@ -24,6 +24,7 @@
from common.util.lock import try_lock, un_lock
from common.util.page_utils import page_desc
from dataset.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State
+from dataset.serializers.common_serializers import create_dataset_index
from embedding.models import SourceType, SearchMode
from smartdoc.conf import PROJECT_DIR
from django.utils.translation import gettext_lazy as _
@@ -238,11 +239,8 @@ def update_status(query_set: QuerySet, taskType: TaskType, state: State):
for key in params_dict:
_value_ = params_dict[key]
exec_sql = exec_sql.replace(key, str(_value_))
- lock.acquire()
- try:
+ with lock:
native_update(query_set, exec_sql)
- finally:
- lock.release()
@staticmethod
def embedding_by_document(document_id, embedding_model: Embeddings, state_list=None):
@@ -272,7 +270,6 @@ def is_the_task_interrupted():
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
State.STARTED)
-
# 根据段落进行向量化处理
page_desc(QuerySet(Paragraph)
.annotate(
@@ -285,6 +282,8 @@ def is_the_task_interrupted():
ListenerManagement.get_aggregation_document_status(
document_id)),
is_the_task_interrupted)
+ # 检查是否存在索引
+ create_dataset_index(document_id=document_id)
except Exception as e:
max_kb_error.error(_('Vectorized document: {document_id} error {error} {traceback}').format(
document_id=document_id, error=str(e), traceback=traceback.format_exc()))
diff --git a/apps/common/forms/__init__.py b/apps/common/forms/__init__.py
index 6095421935b..251f01df092 100644
--- a/apps/common/forms/__init__.py
+++ b/apps/common/forms/__init__.py
@@ -22,3 +22,4 @@
from .radio_card_field import *
from .label import *
from .slider_field import *
+from .switch_field import *
diff --git a/apps/common/forms/switch_field.py b/apps/common/forms/switch_field.py
index 9fa176beea0..ea119c3ecfb 100644
--- a/apps/common/forms/switch_field.py
+++ b/apps/common/forms/switch_field.py
@@ -28,6 +28,6 @@ def __init__(self, label: str or BaseLabel,
@param props_info:
"""
- super().__init__('Switch', label, required, default_value, relation_show_field_dict,
+ super().__init__('SwitchInput', label, required, default_value, relation_show_field_dict,
{},
TriggerType.OPTION_LIST, attrs, props_info)
diff --git a/apps/common/handle/impl/doc_split_handle.py b/apps/common/handle/impl/doc_split_handle.py
index 1df7b6a66e0..4161f13a19d 100644
--- a/apps/common/handle/impl/doc_split_handle.py
+++ b/apps/common/handle/impl/doc_split_handle.py
@@ -112,11 +112,7 @@ def get_image_id(image_id):
title_font_list = [
[36, 100],
- [26, 36],
- [24, 26],
- [22, 24],
- [18, 22],
- [16, 18]
+ [30, 36]
]
@@ -130,7 +126,7 @@ def get_title_level(paragraph: Paragraph):
if len(paragraph.runs) == 1:
font_size = paragraph.runs[0].font.size
pt = font_size.pt
- if pt >= 16:
+ if pt >= 30:
for _value, index in zip(title_font_list, range(len(title_font_list))):
if pt >= _value[0] and pt < _value[1]:
return index + 1
diff --git a/apps/common/handle/impl/table/xls_parse_table_handle.py b/apps/common/handle/impl/table/xls_parse_table_handle.py
index 5609e3e8835..897e347e8a8 100644
--- a/apps/common/handle/impl/table/xls_parse_table_handle.py
+++ b/apps/common/handle/impl/table/xls_parse_table_handle.py
@@ -82,7 +82,10 @@ def get_content(self, file, save_image):
for row in data:
# 将每个单元格中的内容替换换行符为 Document
+
+
+
+
+ 认证
+ 去登录
+
-
-
+
