我上篇文章阐述了我需要进行的评估类型，以及想要了解微调后的 LLM 在从新闻稿中进行结构化数据提取任务中的表现有多好。让我们从我最关心的核心指标准确率开始，之后我们可以深入探讨其他一些评估指标。

这篇文章的标题本可以是：微调模型击败 OpenAI，但评估的实现有点麻烦。这篇文章里有大量隐藏的代码，运行速度也很慢。这是我在微调课程工作期间第一次感受到选择微调的痛点和权衡。我意识到如果没有某种系统来处理这一点，维护的复杂性会逐步增加。但更多细节请见文章末尾！

这是一篇内容丰富的长文。我尽量减少了代码的显示量，但如果你想了解图表或评估是如何进行的，请展开"代码"部分。如果你只想直接查看聚合结果，可以点击这里跳到文章末尾。（要查看本项目的其他博客文章，请点击这里。一些背景信息：我在 Maven 上参加 Hamel Husain / Dan Becker 微调课程，并使用几年前收集和标注的数据进行微调，这些数据对于测试结构化数据提取的效果是个不错的例子。）

这些数据都可以在 Hugging Face Hub 的公开仓库中获取。出于评估的目的，我想使用数据集的测试集，因为我们的模型都没有看过这些数据，所以它很适合用来确定我们的模型在新数据上的表现。

from datasets import load_dataset
import pandas as pd
from rich import print

test_dataset = load_dataset("strickvl/isafpressreleases", split="test")
test_df = pd.DataFrame(test_dataset)

test_dataset

Dataset({
    features: ['name', 'eventrefnumber', 'text', 'StartDate', 'eventtype', 'province', 'citydistrict', 'village', 'targetgroup', 'commander', 'position', 'minkilled', 'mincaptured', 'capturedcharacterisation', 'killedcharacterisation', 'killq', 'captureq', 'killcaptureraid', 'airstrike', 'noshotsfired', 'dataprocessed', 'flagged', 'glossarymeta', 'minleaderskilled', 'minfacilitatorskilled', 'minleaderscaptured', 'minfacilitatorscaptured', 'leaderq'],
    num_rows: 724
})

我们首先在 DataFrame 中添加一个额外的列，然后为数据集中的每一行进行预测。我们会将预测副本存储到该列中，以确保不必重复这个计算密集型步骤。

但首先我们需要将数据组装为 Pydantic 对象，以便进行验证和其他便利功能。

from enum import Enum
from typing import Dict, Set, Annotated, Optional
from pydantic import BaseModel, Field, validator, ValidationInfo
from datetime import date

class EventType(str, Enum):
    airstrike = "airstrike"
    detention = "detention"
    captureandkill = "captureandkill"
    insurgentskilled = "insurgentskilled"
    exchangeoffire = "exchangeoffire"
    civiliancasualty = "civiliancasualty"

class Province(str, Enum):
    badakhshan = "badakhshan"
    badghis = "badghis"
    baghlan = "baghlan"
    balkh = "balkh"
    bamyan = "bamyan"
    day_kundi = "day_kundi"
    farah = "farah"
    faryab = "faryab"
    ghazni = "ghazni"
    ghor = "ghor"
    helmand = "helmand"
    herat = "herat"
    jowzjan = "jowzjan"
    kabul = "kabul"
    kandahar = "kandahar"
    kapisa = "kapisa"
    khost = "khost"
    kunar = "kunar"
    kunduz = "kunduz"
    laghman = "laghman"
    logar = "logar"
    nangarhar = "nangarhar"
    nimroz = "nimroz"
    nuristan = "nuristan"
    paktya = "paktya"
    paktika = "paktika"
    panjshir = "panjshir"
    parwan = "parwan"
    samangan = "samangan"
    sar_e_pul = "sar_e_pul"
    takhar = "takhar"
    uruzgan = "uruzgan"
    wardak = "wardak"
    zabul = "zabul"

class TargetGroup(str, Enum):
    taliban = "taliban"
    haqqani = "haqqani"
    criminals = "criminals"
    aq = "aq"
    hig = "hig"
    let = "let"
    imu = "imu"
    judq = "judq"
    iju = "iju"
    hik = "hik"
    ttp = "ttp"
    other = "other"

def validate_event_type(value: str):
    valid_values = [
        "airstrike",
        "detention",
        "captureandkill",
        "insurgentskilled",
        "exchangeoffire",
        "civiliancasualty",
    ]
    if value.lower() not in valid_values:
        return "other"
    return value.lower()

def validate_province(value: str):
    valid_values = [
        "badakhshan",
        "badghis",
        "baghlan",
        "balkh",
        "bamyan",
        "day_kundi",
        "farah",
        "faryab",
        "ghazni",
        "ghor",
        "helmand",
        "herat",
        "jowzjan",
        "kabul",
        "kandahar",
        "kapisa",
        "khost",
        "kunar",
        "kunduz",
        "laghman",
        "logar",
        "nangarhar",
        "nimroz",
        "nuristan",
        "paktya",
        "paktika",
        "panjshir",
        "parwan",
        "samangan",
        "sar_e_pul",
        "takhar",
        "uruzgan",
        "wardak",
        "zabul",
    ]
    if value.lower() not in valid_values:
        return "other"
    return value.lower()

def validate_target_group(value: str):
    valid_values = [
        "taliban",
        "haqqani",
        "criminals",
        "aq",
        "hig",
        "let",
        "imu",
        "judq",
        "iju",
        "hik",
        "ttp",
        "other",
    ]
    if value.lower() not in valid_values:
        return "other"
    return value.lower()

class IsafEvent(BaseModel):
    name: str = Field(
        description="A title or name for the event which summarises the event as a headline"
    )
    text: Optional[str] = Field(description="The full text of the press release")
    start_date: date = Field(
        description="The start date of the event in YYYY-MM-DD format"
    )
    event_type: Set[Annotated[str, Field(validator=validate_event_type)]] = Field(
        description="The event type. Can be multiple types."
    )
    province: Set[Annotated[str, Field(validator=validate_province)]] = Field(
        description="The province in which the event occurred. Can be multiple provinces."
    )
    target_group: Set[Annotated[str, Field(validator=validate_target_group)]] = Field(
        description="The group that was targetted during the event. Can be multiple groups."
    )
    min_killed: int = Field(
        description="The minimum number of people killed during the event"
    )
    min_captured: int = Field(
        description="The minimum number of people captured during the event"
    )
    kill

以下是我们训练数据的几个例子作为 Pydantic 模型时的样子：

from typing import List

events: List[IsafEvent] = []

for i, row in list(test_df.iterrows()):
    event_types = set(
        eventtype.strip().lower() for eventtype in row["eventtype"].split(",")
    )
    provinces = set(province.strip().lower() for province in row["province"].split(","))
    target_groups = set(
        target_group.strip().lower() for target_group in row["targetgroup"].split(",")
    )

events.append(
        IsafEvent(
            name=row["name"],
            text=row["text"],
            start_date=row["StartDate"].to_pydatetime().date(),
            event_type=event_types,
            province=provinces,
            target_group=target_groups,
            min_killed=int(row["minkilled"]),
            min_captured=int(row["mincaptured"]),
            killq=row["killq"] == "true",
            captureq=row["captureq"] == "true",
            killcaptureraid=row["killcaptureraid"] == "true",
            airstrike=row["airstrike"] == "true",
            noshotsfired=row["noshotsfired"] == "true",
            min_leaders_killed=int(row["minleaderskilled"]),
            min_leaders_captured=int(row["minleaderscaptured"]),
        )
    )

print(events[:2])

[ IsafEvent( name='5', text='2013-01-S-025\n\nKABUL, Afghanistan (Jan. 25, 2013)\nDuring a security operation in Andar district, Ghazni province, yesterday, an Afghan and coalition force killed the Taliban leader, Alaudin. Alaudin oversaw a group of insurgents responsible for conducting remote-controlled improvised explosive device and small-arms fire attacks against Afghan and coalition forces. Prior to his death, Alaudin was planning attacks against Afghan National Police in Ghazni province.', start_date=datetime.date(2013, 1, 24), event_type={'insurgentskilled'}, province={'ghazni'}, target_group={'taliban'}, min_killed=1, min_captured=0, killq=True, captureq=False, killcaptureraid=False, airstrike=False, noshotsfired=False, min_leaders_killed=1, min_leaders_captured=0, predictions={} ), IsafEvent( name='2', text='2011-11-S-034\nISAF Joint Command - Afghanistan\nFor Immediate Release\n\nKABUL, Afghanistan (Nov. 20, 2011)\nA coalition security force detained numerous suspected insurgents during an operation in Marjeh district, Helmand province, yesterday. The force conducted the operation after receiving information that a group of insurgents were at a compound in the area. After calling for the men inside to come out peacefully, the insurgents emerged and were detained without incident.', start_date=datetime.date(2011, 11, 19), event_type={'detention'}, province={'helmand'}, target_group={''}, min_killed=0, min_captured=4, killq=False, captureq=True, killcaptureraid=True, airstrike=False, noshotsfired=False, min_leaders_killed=0, min_leaders_captured=0, predictions={} ) ]

因此，在进行预测时，我们希望模型输出如下所示的 JSON 字符串：

json_str = events[0].model_dump_json(exclude={"text", "predictions"}) print(json_str)

{"name":"5","start_date":"2013-01-24","event_type":["insurgentskilled"],"province":["ghazni"],"target_group":["tali ban"],"min_killed":1,"min_captured":0,"killq":true,"captureq":false,"killcaptureraid":false,"airstrike":false,"nosh otsfired":false,"min_leaders_killed":1,"min_leaders_captured":0}

我先使用 GPT 模型进行完整评估。为了获得不错的结果，我需要一个稍微复杂一些的提示词。我不能直接传入微调模型所使用的完全相同的提示词，因为 GPT 模型并未经过训练或微调，无法针对这些特定提示词作出响应。这其实带来了一个有趣的问题：为了让 GPT 提示词达到与微调模型相同的准确度，我们应该投入多少精力？换句话说，对于必须接受不同提示词的模型，是否真的存在一种能够进行同类比较的方法？

让我们用 OpenAI GPT-4o 和 GPT-4 Turbo 试试看效果如何。你会注意到，为了让 GPT 模型有机会与微调模型抗衡，提示词必须写得很长。理想情况下，我还想在上下文中塞入更多示例，但我也不希望使用的 token 数量急剧膨胀。

from openai import OpenAI from rich import print import json import os

def query_openai(article_text: str, model: str) -> str: query = ( f"The following is a press release issued by ISAF (formerly operating in Afghanistan):\n{article_text}\n\n" "## Extraction request\n" "Please extract the following information from the press release:\n" "- The name of the event (summarising the event / text as a headline)\n" "- The start date of the event\n" "- The event type(s)\n" "- The province(s) in which the event occurred\n" "- The target group(s) of the event\n" "- The minimum number of people killed during the event\n" "- The minimum number of people captured during the event\n" "- Whether someone was killed or not during the event\n" "- Whether someone was captured or not during the event\n" "- Whether the event was a so-called 'kill-capture raid'\n" "- Whether an airstrike was used during the event\n" "- Whether no shots were fired during the event\n" "- The minimum number of leaders killed during the event\n" "- The minimum number of leaders captured during the event\n\n" "## Annotation notes:\n" "- A 'faciliator' is not a leader.\n" "- If a press release states that 'insurgents' were detained without further " "details, assign a minimum number of two detained. Interpret 'a couple' as " "two. Interpret 'several' as at least three, even though it may sometimes " "refer to seven or eight. Classify the terms 'a few', 'some', 'a group', 'a " "small group', and 'multiple' as denoting at least three, even if they " "sometimes refer to larger numbers. Choose the smaller number if no other " "information is available in the press release to come up with a minimally " "acceptable figure. Interpret 'numerous' and 'a handful' as at least four, " "and 'a large number' as at least five.\n\n" "## Example:\n" "Article text: 'ISAF Joint Command Evening Operational Update Feb. 19, 2011\nISAF Joint Command - " "Afghanistan\u20282011-02-S-143\u2028For Immediate Release \u2028\u2028KABUL, Afghanistan (Feb. 19)\u2028\u2028ISAF " "service members at a compound in Sangin district, Helmand province observed numerous insurgents north and south of " "their position talking on radios today. After gaining positive identification of the insurgent positions, the " "coalition troops engaged, killing several insurgents. Later, the ISAF troops observed more insurgents positioning " "in the area with weapons. After positive identification, coalition forces continued firing on the various insurgent " "positions, resulting in several more insurgents being killed.'\n\n" 'Output: {"name":"Several insurgents killed in ' 'Helmand","start_date":"2011-02-18","event_type":["insurgentskilled"],"province":["helmand"],"target_group":[""],"mi' 'n_killed":6,"min_captured":0,"killq":true,"captureq":false,"killcaptureraid":false,"airstrike":false,"noshotsfired"' ':false,"min_leaders_killed":0,"min_leaders_captured":0}' )

set up the prediction harness

client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

response = client.chat.completions.create( model=model, response_format={"type": "json_object"}, messages=[ { "role": "system", "content": "You are an expert at identifying events in a press release. You are precise " "and always make sure you are correct, drawing inference from the text of the " "press release.\n\n You always return a JSON string with the following schema: " "## JSON Schema details\n" "Here is some of the schema for the JSON output string you " "should make use of: event_types =

我们可以用一个简单的示例来确保这个函数正常工作：

json_str = query_openai(events[0].text, "gpt-4o") print(json.loads(json_str))

{ 'name': 'Taliban leader Alaudin killed in Ghazni', 'start_date': '2013-01-24', 'event_type': ['insurgentskilled'], 'province': ['ghazni'], 'target_group': ['taliban'], 'min_killed': 1, 'min_captured': 0, 'killq': True, 'captureq': False, 'killcaptureraid': True, 'airstrike': False, 'noshotsfired': False, 'min_leaders_killed': 1, 'min_leaders_captured': 0 }

我们的模型运行正常（正如预期），而且也成功返回了一个 JSON 字符串。接下来，我们来编写一段代码，遍历所有测试数据、获取预测结果，然后将这些预测结果存储到 Pydantic 对象中。

对于批量预测，我们会确保以异步方式执行，因为事件数量很多，我们可不想等上一整天。你还会看到，为了应对 GPT-3.5-turbo 模型的速率限制，我不得不在函数中加入一些重试逻辑。

make async work within a notebook

import nest_asyncio

nest_asyncio.apply()

import aiohttp import asyncio from typing import List from openai import OpenAI

async def async_query_openai(
    session,
    article_text: str,
    model: str,
    max_retries: int = 3,
    retry_delay: float = 1.0,
) -> str:
    query = (
        f"The following is a press release issued by ISAF (formerly operating in Afghanistan):\n{article_text}\n\n"
        "## Extraction request\n"
        "Please extract the following information from the press release:\n"
        "- The name of the event (summarising the event / text as a headline)\n"
        "- The start date of the event\n"
        "- The event type(s)\n"
        "- The province(s) in which the event occurred\n"
        "- The target group(s) of the event\n"
        "- The minimum number of people killed during the event\n"
        "- The minimum number of people captured during the event\n"
        "- Whether someone was killed or not during the event\n"
        "- Whether someone was captured or not during the event\n"
        "- Whether the event was a so-called 'kill-capture raid'\n"
        "- Whether an airstrike was used during the event\n"
        "- Whether no shots were fired during the event\n"
        "- The minimum number of leaders killed during the event\n"
        "- The minimum number of leaders captured during the event\n\n"
        "## Annotation notes:\n"
        "- A 'faciliator' is not a leader.\n"
        "- If a press release states that 'insurgents' were detained without further "
        "details, assign a minimum number of two detained. Interpret 'a couple' as "
        "two. Interpret 'several' as at least three, even though it may sometimes "
        "refer to seven or eight. Classify the terms 'a few', 'some', 'a group', 'a "
        "small group', and 'multiple' as denoting at least three, even if they "
        "sometimes refer to larger numbers. Choose the smaller number if no other "
        "information is available in the press release to come up with a minimally "
        "acceptable figure. Interpret 'numerous' and 'a handful' as at least four, "
        "and 'a large number' as at least five.\n\n"
        "## Example:\n"
        "Article text: 'ISAF Joint Command Evening Operational Update Feb. 19, 2011\nISAF Joint Command - "
        "Afghanistan\u20282011-02-S-143\u2028For Immediate Release \u2028\u2028KABUL, Afghanistan (Feb. 19)\u2028\u2028ISAF "
        "service members at a compound in Sangin district, Helmand province observed numerous insurgents north and south of "
        "their position talking on radios today. After gaining positive identification of the insurgent positions, the "
        "coalition troops engaged, killing several insurgents. Later, the ISAF troops observed more insurgents positioning "
        "in the area with weapons. After positive identification, coalition forces continued firing on the various insurgent "
        "positions, resulting in several more insurgents being killed.'\n\n"
        'Output: `{"name":"Several insurgents killed in '
        'Helmand","start_date":"2011-02-18","event_type":["insurgentskilled"],"province":["helmand"],"target_group":[""],"mi'
        'n_killed":6,"min_captured":0,"killq":true,"captureq":false,"killcaptureraid":false,"airstrike":false,"noshotsfired"'
        ':false,"min_leaders_killed":0,"min_leaders_captured":0}`'
    )

client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

retries = 0
    while retries < max_retries:
        async with session.post(
            "https://api.openai.com/v1/chat/completions",
            headers={"Authorization": f"Bearer {client.api_key}"},
            json={
                "model": model,
                "response_format": {"type": "json_object"},
                "messages": [
                    {
                        "role": "system",
                        "content": "You are an expert at identifying events in a press release. Yo

现在你可以看到，每个事件都附有三个预测结果。

print(events[0])

IsafEvent(
    name='5',
    text='2013-01-S-025\n\nKABUL, Afghanistan (Jan. 25, 2013)\nDuring a security operation in Andar district, 
Ghazni province, yesterday, an Afghan and coalition force killed the Taliban leader, Alaudin. Alaudin oversaw a 
group of insurgents responsible for conducting remote-controlled improvised explosive device and small-arms fire 
attacks against Afghan and coalition forces. Prior to his death, Alaudin was planning attacks against Afghan 
National Police in Ghazni province.',
    start_date=datetime.date(2013, 1, 24),
    event_type={'insurgentskilled'},
    province={'ghazni'},
    target_group={'taliban'},
    min_killed=1,
    min_captured=0,
    killq=True,
    captureq=False,
    killcaptureraid=False,
    airstrike=False,
    noshotsfired=False,
    min_leaders_killed=1,
    min_leaders_captured=0,
    predictions={
        'gpt-4o': '{\n  "name": "Taliban leader Alaudin killed in Ghazni",\n  "start_date": "2013-01-24",\n  
"event_type": ["insurgentskilled", "captureandkill"],\n  "province": ["ghazni"],\n  "target_group": ["taliban"],\n 
"min_killed": 1,\n  "min_captured": 0,\n  "killq": true,\n  "captureq": false,\n  "killcaptureraid": true,\n  
"airstrike": false,\n  "noshotsfired": false,\n  "min_leaders_killed": 1,\n  "min_leaders_captured": 0\n}',
        'gpt-4-turbo': '{\n    "name": "Taliban leader Alaudin killed in Ghazni",\n    "start_date": 
"2013-01-24",\n    "event_type": ["captureandkill"],\n    "province": ["ghazni"],\n    "target_group": 
["taliban"],\n    "min_killed": 1,\n    "min_captured": 0,\n    "killq": true,\n    "captureq": false,\n    
"killcaptureraid": true,\n    "airstrike": false,\n    "noshotsfired": false,\n    "min_leaders_killed": 1,\n    
"min_leaders_captured": 0\n}',
        'gpt-3.5-turbo': '{\n    "name": "Taliban leader Alaudin killed in Ghazni province",\n    "start_date": 
"2013-01-24",\n    "event_type": ["captureandkill"],\n    "province": ["ghazni"],\n    "target_group": 
["taliban"],\n    "min_killed": 1,\n    "min_captured": 0,\n    "killq": true,\n    "captureq": false,\n    
"killcaptureraid": false,\n    "airstrike": false,\n    "noshotsfired": false,\n    "min_leaders_killed": 1,\n    
"min_leaders_captured": 0\n}'
    }
)

目前所有这些预测结果都保存在内存中，因此现在或许正适合将它们写入数据集并推送到 Hugging Face Hub，以防 notebook 崩溃、本地计算机关机或发生其他意外情况。

我会创建一个函数来处理这件事，因为之后还要对其他模型重复这一过程。它有点冗长，但我认为这样更合适，便于你看清楚具体发生了什么。

from datasets import Dataset

def convert_to_dataset(data: List[IsafEvent]) -> Dataset:
    names = []
    texts = []
    start_dates = []
    provinces = []
    target_groups = []
    event_types = []
    predictions = []
    min_killeds = []
    min_captureds = []
    killqs = []
    captureqs = []
    killcaptureraids = []
    airstrikes = []
    noshotsfireds = []
    min_leaders_killeds = []
    min_leaders_captureds = []

for item in data:
        names.append(item.name)
        texts.append(item.text)
        start_dates.append(item.start_date)
        provinces.append(item.province)
        target_groups.append(item.target_group)
        event_types.append(item.event_type)
        predictions.append(item.predictions)
        min_killeds = []