Langchain 加载网络信息实现RAG以及UnstructuredURLLoader的使用_langchain 读取网页 rag

以下实现了从 wikipedia 加载 Android 的网页然后保存在本地的向量数据库，然后通过上下文发给大模型，让他来总结什么是android 。

from langchain_community.vectorstores import Chroma
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_community.embeddings import JinaEmbeddings
from langchain_community.llms import Tongyi
from langchain_core.runnables import RunnableParallel, RunnablePassthrough
import os
os.environ["DASHSCOPE_API_KEY"] = "sk-cc1c8314fdbd43ceaf26ec1824d5dd3b"
llm = Tongyi()

from langchain_community.document_loaders import UnstructuredURLLoader

urls = [
    "https://en.wikipedia.org/wiki/Android_(operating_system)",
    # "https://answer.baidu.com/answer/land?params=MSZrNRfsun5P549PG8zUdV6PXibobtP6242M%2FwAkrm0sWWOs5IAFrz6XAVTZu6sZDpxyjT4AEJir6bCqxEWwPoy%2F7dCKyABf%2BFgxpeKWkX0isoUgqs7ViRSvL3B%2BBsajzbX1Ai05uEVz4Owgwf361B4xj1CwAAbsAD3PBqPnJT4%3D&from=dqa&lid=ebb4fc0600ddb6f0&word=%E4%BB%80%E4%B9%88%E6%98%AF%E8%8B%B9%E6%9E%9C",
    # "https://www.understandingwar.org/backgrounder/russian-offensive-campaign-assessment-february-8-2023",
    # "https://blog.csdn.net/oHeHui1/article/details/136261119?spm=1001.2014.3001.5502"
]

loader = UnstructuredURLLoader(urls=urls)
documents = loader.load_and_split()
print(documents)
embeddings = JinaEmbeddings(
    jina_api_key="jina_c5d02a61c97d4d79b88234362726e94aVLMTvF38wvrElYqpGYSxFtC5Ifhj", model_name="jina-embeddings-v2-base-en"
)

# # 第一次存入本地
vectorstore = Chroma.from_documents(documents, embeddings,persist_directory="./wikipedia")

# # 从本地加载
# vectorstore = Chroma(persist_directory="./wikipedia", embedding_function=embeddings)

retriever = vectorstore.as_retriever()
template = """Answer the question based on the context below. If the
question cannot be answered using the information provided answer
with "I don't know"     

Context: {context}

Question: {question}
"""
prompt = ChatPromptTemplate.from_template(template)
llm = ChatGLM.ChatGLM_LLM(verbose=False)
output_parser = StrOutputParser()
setup_and_retrieval = RunnableParallel(
    {"context": retriever, "question": RunnablePassthrough()}
)
chain = setup_and_retrieval | prompt | llm | output_parser
# print(chain.invoke("苹果是什么"))
# print(chain.invoke("苹果有哪些功效"))
# print(chain.invoke("如何创建虚拟环境"))
print(chain.invoke("what is android"))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52