上传文件至 /

kk.py

@@ -0,0 +1,55 @@
+from sklearn.metrics import cohen_kappa_score
+
+annotations_a = ['positive', 'negative', 'positive', 'positive',
+                 'negative'] * 20
+annotations_b = ['positive', 'negative', 'positive', 'positive',
+                 'negative'] * 20
+
+kappa = cohen_kappa_score(annotations_a, annotations_b)
+print(f"Kappa = {kappa:.3f}")
+# 两人完全一致 → Kappa = 1.000
+# 不同结果示例
+annotations_a = ['positive'] * 60 + ['negative'] * 40
+annotations_b = ['positive'] * 55 + ['negative'] * 45
+
+kappa = cohen_kappa_score(annotations_a, annotations_b)
+print(f"Kappa = {kappa:.3f}")
+# Kappa ≈ 0.69
+def ner_f1(entities_a, entities_b):
+    """entities 是 [(text, type, start, end), ...] 列表"""
+    set_a = set(entities_a)
+    set_b = set(entities_b)
+
+    tp = len(set_a & set_b)  # 都标了
+    fp = len(set_a - set_b)  # A 标了 B 没标
+    fn = len(set_b - set_a)  # B 标了 A 没标
+
+    precision = tp / (tp + fp) if (tp + fp) > 0 else 0
+    recall = tp / (tp + fn) if (tp + fn) > 0 else 0
+    f1 = 2 * precision * recall / (precision + recall) \
+        if (precision + recall) > 0 else 0
+
+    return precision, recall, f1
+
+
+# 例
+a = [("张伟", "PER", 0, 2), ("清华", "ORG", 5, 7)]
+b = [("张伟", "PER", 0, 2), ("清华", "ORG", 5, 7), ("北京", "LOC", 10, 12)]
+
+p, r, f1 = ner_f1(a, b)
+print(f"P={p:.2f} R={r:.2f} F1={f1:.2f}")
+# P=1.00 R=0.67 F1=0.80
+from statsmodels.stats.inter_rater import fleiss_kappa
+
+# 每个任务每个类别有多少人标
+# 例: 5个任务, 3个类别, 4人标注
+counts = [
+    [4, 0, 0],   # 任务1: 4人都标类别0
+    [0, 4, 0],   # 任务2: 4人都标类别1
+    [2, 2, 0],   # 任务3: 2人标0, 2人标1
+    [1, 1, 2],   # 任务4: 各有分歧
+    [0, 0, 4],   # 任务5: 4人都标类别2
+]
+
+kappa = fleiss_kappa(counts)
+print(f"Fleiss Kappa = {kappa:.3f}")