ローカルLLMでWeb scraping

ローカルLLMは情報の流れが速いので、気になるWebページを後で読もうとブックマークしていたのですが・・・

気が付くと数百件のブックマークになってしまいました。
有用と思った情報を抽出したのですから、ここから検索したいですね。
ローカルLLMでどこまでできるか試してみましょう。

環境をつくって・・・

python -m venv .venv && 
source .venv/bin/activate && 
pip install -U pip packaging setuptools wheel
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 &&
pip install langchain playwright beautifulsoup4 &&
pip install html2text chromadb

playwright install

RetrievalQAを使用してみます。今回は近傍探索にChromaを使用しています。Langchainの既定のベクターストアで、オープンソースです。

from langchain.tools import DuckDuckGoSearchRun
from langchain.agents import Tool, initialize_agent
from langchain.chains import LLMMathChain
from pydantic import BaseModel, Field

from langchain.llms import HuggingFacePipeline
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline


import os
from config import HUGGINGFACEHUB_API_TOKEN
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
from langchain.vectorstores import Chroma

from langchain.document_loaders import AsyncHtmlLoader
from langchain.document_transformers import Html2TextTransformer
from langchain.text_splitter import RecursiveCharacterTextSplitter


# LLM
model_name_or_path = "TheBloke/openchat_3.5-GPTQ"
# To use a different branch, change revision
# For example: revision="gptq-4bit-32g-actorder_True"
model = AutoModelForCausalLM.from_pretrained(
    model_name_or_path,
    device_map="auto",
    trust_remote_code=False,
    revision="gptq-8bit-32g-actorder_True",
)
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)

pipe = pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
    max_new_tokens=512,
    do_sample=True,
    temperature=0.1,
    top_p=0.95,
    top_k=40,
    repetition_penalty=1.1,
)

llm = HuggingFacePipeline(pipeline=pipe)

os.environ["HUGGINGFACEHUB_API_TOKEN"] = HUGGINGFACEHUB_API_TOKEN
embeddings = HuggingFaceEmbeddings()


urls = [
    "https://note.com/kioju/n/n3d3401cd499f",
    "https://note.com/kioju/n/n6309bcf6f3af",
    "https://note.com/kioju/n/ne470b3b6b72e",
    "https://note.com/kioju/n/n9b43c8aa654c",
    "https://note.com/kioju/n/n6309bcf6f3af",
    "https://note.com/kioju/n/n93fcbfed878b",
    "https://note.com/kioju/n/n73ed761e2a15",
]
loader = AsyncHtmlLoader(urls)
contents = loader.load()


# pip install html2text
html2text = Html2TextTransformer()
docs = html2text.transform_documents(contents)


text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=200,
    chunk_overlap=50,
)
texts = text_splitter.split_documents(docs)

# pip install chromadb
vectorstore = Chroma.from_documents(texts, embeddings, persist_directory="./chroma_db_oai")

# ベクターストアを検索エンジンとして使用
retriever = vectorstore.as_retriever()

from langchain.chains import RetrievalQA

# チェーンを作り、それを使って質問に答える
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever)

query = "Qwenについて教えてください"
answer = qa.run(query)
print(answer)

Qwenは、Qwen-72B-Chat-Int4と呼ばれるものがあります。それは48GBのメモリを持ち、仮想環境を作成することができます。

Agentとして推敲させることで、もうすこし良い応答が得られる可能性があります。試してみましょう。

from langchain.tools import DuckDuckGoSearchRun
from langchain.agents import Tool, initialize_agent
from langchain.chains import LLMMathChain
from pydantic import BaseModel, Field

from langchain.llms import HuggingFacePipeline
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline


import os
from config import HUGGINGFACEHUB_API_TOKEN
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import FAISS
from langchain.vectorstores import Chroma

from langchain.document_loaders import AsyncHtmlLoader
from langchain.document_transformers import Html2TextTransformer
from langchain.text_splitter import RecursiveCharacterTextSplitter


# LLM
model_name_or_path = "TheBloke/openchat_3.5-GPTQ"
# To use a different branch, change revision
# For example: revision="gptq-4bit-32g-actorder_True"
model = AutoModelForCausalLM.from_pretrained(
    model_name_or_path,
    device_map="auto",
    trust_remote_code=False,
    revision="gptq-8bit-32g-actorder_True",
)
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)

pipe = pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
    max_new_tokens=512,
    do_sample=True,
    temperature=0.1,
    top_p=0.95,
    top_k=40,
    repetition_penalty=1.1,
)

llm = HuggingFacePipeline(pipeline=pipe)

os.environ["HUGGINGFACEHUB_API_TOKEN"] = HUGGINGFACEHUB_API_TOKEN
embeddings = HuggingFaceEmbeddings()


urls = [
    "https://note.com/kioju/n/n3d3401cd499f",
    "https://note.com/kioju/n/n6309bcf6f3af",
    "https://note.com/kioju/n/ne470b3b6b72e",
    "https://note.com/kioju/n/n9b43c8aa654c",
    "https://note.com/kioju/n/n6309bcf6f3af",
    "https://note.com/kioju/n/n93fcbfed878b",
    "https://note.com/kioju/n/n73ed761e2a15",
]
loader = AsyncHtmlLoader(urls)
contents = loader.load()


# pip install html2text
html2text = Html2TextTransformer()
docs = html2text.transform_documents(contents)


text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=200,
    chunk_overlap=50,
)
texts = text_splitter.split_documents(docs)

# pip install chromadb
vectorstore = Chroma.from_documents(texts, embeddings, persist_directory="./chroma_db_oai")

# ベクターストアを検索エンジンとして使用
retriever = vectorstore.as_retriever()

# Tools
from langchain.agents.agent_toolkits import create_retriever_tool
tool = create_retriever_tool(
    retriever,
    "search-web-scrap",
    "Search for scraps of Web pages of particular interest.",
)
tools = [tool]

# agent
agent = initialize_agent(
    tools,
    llm,
    agent="zero-shot-react-description",
    verbose=True,
    handle_parsing_errors=True,
)

text = "Qwenについて教えてください"
output = agent.run(text)
output = output.split("\n")[0]
print(output)

Qwen is a large language model with 72 billion parameters. It requires at least 48 GB of GPU memory to run. It was released by Hugging Face and has been used successfully on Windows 11 with an RTX 4090 GPU.

要点をまとめた良い応答となりました。