Week 4. 지식 추출 파이프라인

원시 텍스트 → NER → Entity Linking → 관계 추출 → 지식 그래프
    │           │          │              │
    │           ↓          ↓              ↓
    │      [개체들]     [URIs]       [트리플]
    │       "놀란"   ex:Nolan   (Nolan, directed, Inception)

import spacy
 
nlp = spacy.load("ko_core_news_lg")  # 한국어 모델
 
text = "크리스토퍼 놀란이 2010년에 인셉션을 감독했다."
doc = nlp(text)
 
for ent in doc.ents:
    print(f"{ent.text} ({ent.label_})")
# 크리스토퍼 놀란 (PERSON)
# 2010년 (DATE)
# 인셉션 (WORK_OF_ART)

from openai import OpenAI
 
client = OpenAI()
 
def extract_entities(text):
    response = client.chat.completions.create(
        model="gpt-4",
        messages=[{
            "role": "system",
            "content": """텍스트에서 개체를 추출하세요.
            JSON 형식: [{"text": "", "type": "", "description": ""}]
            유형: PERSON, MOVIE, ORGANIZATION, DATE, LOCATION"""
        }, {
            "role": "user",
            "content": text
        }]
    )
    return json.loads(response.choices[0].message.content)
 
entities = extract_entities("크리스토퍼 놀란이 워너 브라더스에서 인셉션을 감독했다.")
# [
#   {"text": "크리스토퍼 놀란", "type": "PERSON", "description": "감독"},
#   {"text": "인셉션", "type": "MOVIE", "description": "2010년 영화"},
#   {"text": "워너 브라더스", "type": "ORGANIZATION", "description": "제작사"}
# ]

def extract_relations(text, entities):
    entity_list = ", ".join([e["text"] for e in entities])
 
    response = client.chat.completions.create(
        model="gpt-4",
        messages=[{
            "role": "system",
            "content": f"""주어진 개체: {entity_list}
            텍스트에서 관계를 추출하세요.
            JSON 형식: [{{"subject": "", "predicate": "", "object": ""}}]"""
        }, {
            "role": "user",
            "content": text
        }]
    )
    return json.loads(response.choices[0].message.content)
 
# 결과:
# [{"subject": "크리스토퍼 놀란", "predicate": "감독하다", "object": "인셉션"}]

from transformers import pipeline
 
# REBEL 모델 사용 (Relation Extraction By End-to-end Language generation)
triplet_extractor = pipeline(
    "text2text-generation",
    model="Babelscape/rebel-large",
    tokenizer="Babelscape/rebel-large"
)
 
text = "Christopher Nolan directed Inception."
output = triplet_extractor(text, return_tensors=True, max_length=256)
# 출력 파싱하여 트리플 추출

소스 A: "크리스 놀란"
소스 B: "크리스토퍼 놀란"
소스 C: "C. Nolan"
→ 모두 같은 사람!

from rapidfuzz import fuzz
 
def find_similar_entities(new_entity, existing_entities, threshold=85):
    matches = []
    for existing in existing_entities:
        score = fuzz.ratio(new_entity.lower(), existing.lower())
        if score >= threshold:
            matches.append((existing, score))
    return sorted(matches, key=lambda x: x[1], reverse=True)
 
# 임베딩 기반 접근
from sentence_transformers import SentenceTransformer
 
model = SentenceTransformer('all-MiniLM-L6-v2')
 
def entity_similarity(entity1, entity2):
    emb1 = model.encode(entity1)
    emb2 = model.encode(entity2)
    return cosine_similarity([emb1], [emb2])[0][0]

class KnowledgeExtractor:
    def __init__(self, llm_client, entity_store):
        self.llm = llm_client
        self.store = entity_store
 
    def extract(self, text):
        # 1. NER
        entities = self.extract_entities(text)
 
        # 2. Entity Resolution
        resolved_entities = []
        for entity in entities:
            existing = self.store.find_similar(entity["text"])
            if existing:
                entity["uri"] = existing.uri
            else:
                entity["uri"] = self.store.create_new(entity)
            resolved_entities.append(entity)
 
        # 3. 관계 추출
        relations = self.extract_relations(text, resolved_entities)
 
        # 4. 트리플 구성
        triples = []
        for rel in relations:
            subj_uri = self.get_uri(rel["subject"], resolved_entities)
            obj_uri = self.get_uri(rel["object"], resolved_entities)
            triples.append((subj_uri, rel["predicate"], obj_uri))
 
        return triples

위키피디아 텍스트
    ↓ NER (개체명 인식)
[Christopher Nolan] [Inception] [2010]
    ↓ Relation Extraction (관계 추출)
(Nolan) --directed--> (Inception)
    ↓ Entity Resolution (개체 연결)
Nolan = dbpedia:Christopher_Nolan
    ↓ RDF 트리플 생성