forked from PaddlePaddle/book
-
Notifications
You must be signed in to change notification settings - Fork 0
/
ngram.py
84 lines (70 loc) · 2.43 KB
/
ngram.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from paddle.trainer_config_helpers import *
import math
#################### Data Configure ####################
args = {
'srcText': 'data/simple-examples/data/ptb.train.txt',
'dictfile': 'data/vocabulary.txt'
}
define_py_data_sources2(
train_list="data/train.list",
test_list="data/test.list",
module="dataprovider",
obj="process",
args=args)
settings(
batch_size=100, regularization=L2Regularization(8e-4), learning_rate=3e-3)
dictsize = 1953
embsize = 32
hiddensize = 256
firstword = data_layer(name="firstw", size=dictsize)
secondword = data_layer(name="secondw", size=dictsize)
thirdword = data_layer(name="thirdw", size=dictsize)
fourthword = data_layer(name="fourthw", size=dictsize)
nextword = data_layer(name="fifthw", size=dictsize)
# construct word embedding for each datalayer
def wordemb(inlayer):
wordemb = table_projection(
input=inlayer,
size=embsize,
param_attr=ParamAttr(
name="_proj",
initial_std=0.001,
learning_rate=1,
l2_rate=0, ))
return wordemb
Efirst = wordemb(firstword)
Esecond = wordemb(secondword)
Ethird = wordemb(thirdword)
Efourth = wordemb(fourthword)
# concatentate Ngram embeddings into context embedding
contextemb = concat_layer(input=[Efirst, Esecond, Ethird, Efourth])
hidden1 = fc_layer(
input=contextemb,
size=hiddensize,
act=SigmoidActivation(),
layer_attr=ExtraAttr(drop_rate=0.5),
bias_attr=ParamAttr(learning_rate=2),
param_attr=ParamAttr(
initial_std=1. / math.sqrt(embsize * 8), learning_rate=1))
# use context embedding to predict nextword
predictword = fc_layer(
input=hidden1,
size=dictsize,
bias_attr=ParamAttr(learning_rate=2),
act=SoftmaxActivation())
cost = classification_cost(input=predictword, label=nextword)
# network input and output
outputs(cost)