羲和数据集收集器1.4

为了确保代码能够正确地提取并保存 QA 对，我们需要进一步调试和优化。特别是要确保 extract_qa_pairs_from_content 函数能够正确识别和处理不同格式的 QA 对，并且 write_qa_pairs_to_file 函数能够正确地将 QA 对写入文件。

以下是进一步完善的代码：

import os
import json
import fitz  # PyMuPDF
import docx
import re
import tkinter as tk
from tkinter import filedialog, messagebox, simpledialog
import logging# 设置日志记录
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')def clean_text(text):""" 清理文本，去除多余的空白字符和标点符号，替换影响数据集的特殊字符 """text = re.sub(r'\s+', ' ', text)  # 合并多个空白字符text = re.sub(r'[^\w\s]', '', text)  # 去除标点符号text = re.sub(r'"', "'", text)  # 替换双引号text = re.sub(r'\\', '/', text)  # 替换反斜杠return text.strip()def extract_qa_pairs_from_txt(file_path):with open(file_path, 'r', encoding='utf-8') as f:content = f.read()content = clean_text(content)qa_pairs = extract_qa_pairs_from_content(content)return qa_pairsdef extract_qa_pairs_from_docx(file_path):doc = docx.Document(file_path)content = []for para in doc.paragraphs:content.append(para.text)for table in doc.tables:for row in table.rows:for cell in row.cells:content.append(cell.text)for rel in doc.part.rels.values():if "textBox" in rel.target_ref:text_box = rel.target_partfor element in text_box.element.body:if element.tag.endswith('p'):content.append(element.text)content = '\n'.join(content)content = clean_text(content)qa_pairs = extract_qa_pairs_from_content(content)return qa_pairsdef extract_qa_pairs_from_pdf(file_path)