-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathLR.py
100 lines (70 loc) · 24.1 KB
/
LR.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# coding=utf-8
from pyspark.sql import SparkSession
spark = SparkSession \
.builder \
.appName("user") \
.config("spark.some.config.option", "some-value") \
.getOrCreate()
spark.sparkContext.setLogLevel('WARN')
# 一、构造训练集，用户和文章特征作为训练集特征
# 1、clicked作为训练集标签
# 处理文本导致的字符串问题
def article_basic(row):
row = row.split('\x01')
for i in range(len(row)):
if row[i] == 'false':
row[i] = False
elif row[i] == 'true':
row[i] = True
return row[0], row[2], row[3], row[5]
user_article_basic = spark.sparkContext.textFile(r'F:\2019最新 Python黑马头条推荐系统项目1\data\backup\profile.db\user_article_basic')
user_article_basic = user_article_basic.map(article_basic).filter(lambda x: x[2] != '0').toDF(
['user_id', 'article_id', 'channel_id', 'clicked'])
# 2.联合用户特征和文章特征
article_feature = spark.read.parquet('datas/article_features')
user_feature = spark.read.parquet('datas/user_features')
user_article_feature = user_article_basic.join(article_feature, how='inner', on=['article_id'])
user_article_feature = user_article_feature.join(user_feature, how='inner', on=['user_id'])
print("用户及文章特征数据", user_article_feature.take(10))
def feature(row):
user_features = row.user_features[int(row.channel_id)]
clicked = int(row.clicked)
return row.article_features, user_features, clicked
train = user_article_feature.rdd.map(feature)
# 转化为Vectors
def list_to_vector(row):
from pyspark.ml.linalg import Vectors
return Vectors.dense(row[0]), Vectors.dense(row[1]), row[2]
columns = ['article_feature', 'user_feature', 'clicked']
train = train.map(list_to_vector).toDF(columns)
# 3.训练数据特征列和标签列
from pyspark.ml.feature import VectorAssembler
# 文章特征和用户特征合并为一列
train = VectorAssembler().setInputCols(columns[0:2]).setOutputCol('features').transform(train)
print("训练集", train.take(10))
train.coalesce(2).write.parquet(path='datas/train/', mode='overwrite')
# 二、模型训练部分
from pyspark.ml.classification import LogisticRegression
lr = LogisticRegression()
model = lr.setLabelCol("clicked").setFeaturesCol("features").fit(train)
model.write().overwrite().save("models/lr.obj")
# 三、模型预测部分，此处用的是训练集
from pyspark.ml.classification import LogisticRegressionModel
online_model = LogisticRegressionModel.load("models/lr.obj")
sort_res = online_model.transform(train)
print("在线预测结果", sort_res.take(10))
# [Row(article_feature=DenseVector([18.0, 2.6818, 1.1523, 0.6008, 0.4045, 0.3961, 0.2916, 0.2907, 0.2137, 0.181, 0.1622, 0.0952, 0.0548, 0.0051, 0.1187, -0.0195, 0.0497, 0.0017, 0.0951, -0.1054, -0.0287, -0.1153, 0.0491, 0.0531, 0.1113, -0.0432, 0.0399, -0.0116, 0.0139, 0.0195, -0.0027, -0.0225, 0.034, -0.0267, 0.0199, -0.1052, -0.0768, 0.0753, 0.0743, 0.0392, -0.0944, 0.0154, 0.0333, 0.1258, 0.0852, -0.0518, 0.0318, -0.0513, -0.0517, 0.0718, 0.0068, 0.1159, -0.0823, -0.0281, 0.0318, 0.0056, 0.0493, 0.1118, 0.0596, -0.066, 0.0297, -0.0364, -0.045, 0.0716, 0.0975, 0.0062, 0.0367, 0.0108, -0.0409, 0.0174, 0.075, 0.0233, -0.0755, 0.0177, -0.1275, -0.0226, 0.0614, -0.0488, -0.0264, 0.0598, -0.0444, 0.0054, -0.0308, -0.0032, -0.0712, 0.0697, 0.0414, 0.0359, -0.0227, 0.0804, 0.0185, 0.0729, -0.071, 0.0405, -0.0053, 0.0092, 0.0264, 0.0078, -0.0287, -0.0011, 0.0963, 0.0528, 0.075, 0.1261, 0.0403, 0.0505, -0.0356, -0.0353, -0.1071, -0.0002, 0.131]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=0, features=DenseVector([18.0, 2.6818, 1.1523, 0.6008, 0.4045, 0.3961, 0.2916, 0.2907, 0.2137, 0.181, 0.1622, 0.0952, 0.0548, 0.0051, 0.1187, -0.0195, 0.0497, 0.0017, 0.0951, -0.1054, -0.0287, -0.1153, 0.0491, 0.0531, 0.1113, -0.0432, 0.0399, -0.0116, 0.0139, 0.0195, -0.0027, -0.0225, 0.034, -0.0267, 0.0199, -0.1052, -0.0768, 0.0753, 0.0743, 0.0392, -0.0944, 0.0154, 0.0333, 0.1258, 0.0852, -0.0518, 0.0318, -0.0513, -0.0517, 0.0718, 0.0068, 0.1159, -0.0823, -0.0281, 0.0318, 0.0056, 0.0493, 0.1118, 0.0596, -0.066, 0.0297, -0.0364, -0.045, 0.0716, 0.0975, 0.0062, 0.0367, 0.0108, -0.0409, 0.0174, 0.075, 0.0233, -0.0755, 0.0177, -0.1275, -0.0226, 0.0614, -0.0488, -0.0264, 0.0598, -0.0444, 0.0054, -0.0308, -0.0032, -0.0712, 0.0697, 0.0414, 0.0359, -0.0227, 0.0804, 0.0185, 0.0729, -0.071, 0.0405, -0.0053, 0.0092, 0.0264, 0.0078, -0.0287, -0.0011, 0.0963, 0.0528, 0.075, 0.1261, 0.0403, 0.0505, -0.0356, -0.0353, -0.1071, -0.0002, 0.131, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([27.8459, -27.8459]), probability=DenseVector([1.0, 0.0]), prediction=0.0), Row(article_feature=DenseVector([18.0, 2.6235, 1.197, 0.6777, 0.6651, 0.5384, 0.2576, 0.2472, 0.2145, 0.1897, 0.1892, 0.0629, 0.0534, 0.0006, 0.0964, 0.005, 0.0277, 0.0179, 0.0931, -0.0783, -0.021, -0.1023, 0.0335, 0.0529, 0.0946, -0.0218, 0.0114, -0.0483, -0.0091, 0.0057, 0.0119, -0.0022, 0.0289, -0.02, 0.0094, -0.0828, -0.0927, 0.0773, 0.0439, 0.038, -0.1206, -0.0014, 0.0652, 0.1252, 0.0591, -0.0521, 0.0357, -0.0923, -0.0615, 0.0352, 0.011, 0.1025, -0.0652, -0.0455, 0.0124, 0.0183, 0.0181, 0.0767, 0.0859, -0.0402, 0.0345, -0.0224, -0.0086, 0.0752, 0.0809, 0.034, 0.0387, 0.0405, -0.0204, -0.0043, 0.0448, 0.0543, -0.0899, 0.0032, -0.1267, -0.0149, 0.0631, -0.0358, -0.0221, 0.045, -0.0535, -0.0358, -0.0421, 0.007, -0.0707, 0.045, 0.0162, 0.073, 0.0122, 0.081, 0.001, 0.0723, -0.0354, 0.036, -0.0104, 0.0219, 0.0311, -0.0039, -0.036, -0.0257, 0.0649, 0.0384, 0.0787, 0.0789, 0.0136, 0.0142, -0.045, -0.0333, -0.08, 0.0262, 0.081]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=1, features=DenseVector([18.0, 2.6235, 1.197, 0.6777, 0.6651, 0.5384, 0.2576, 0.2472, 0.2145, 0.1897, 0.1892, 0.0629, 0.0534, 0.0006, 0.0964, 0.005, 0.0277, 0.0179, 0.0931, -0.0783, -0.021, -0.1023, 0.0335, 0.0529, 0.0946, -0.0218, 0.0114, -0.0483, -0.0091, 0.0057, 0.0119, -0.0022, 0.0289, -0.02, 0.0094, -0.0828, -0.0927, 0.0773, 0.0439, 0.038, -0.1206, -0.0014, 0.0652, 0.1252, 0.0591, -0.0521, 0.0357, -0.0923, -0.0615, 0.0352, 0.011, 0.1025, -0.0652, -0.0455, 0.0124, 0.0183, 0.0181, 0.0767, 0.0859, -0.0402, 0.0345, -0.0224, -0.0086, 0.0752, 0.0809, 0.034, 0.0387, 0.0405, -0.0204, -0.0043, 0.0448, 0.0543, -0.0899, 0.0032, -0.1267, -0.0149, 0.0631, -0.0358, -0.0221, 0.045, -0.0535, -0.0358, -0.0421, 0.007, -0.0707, 0.045, 0.0162, 0.073, 0.0122, 0.081, 0.001, 0.0723, -0.0354, 0.036, -0.0104, 0.0219, 0.0311, -0.0039, -0.036, -0.0257, 0.0649, 0.0384, 0.0787, 0.0789, 0.0136, 0.0142, -0.045, -0.0333, -0.08, 0.0262, 0.081, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([-26.9755, 26.9755]), probability=DenseVector([0.0, 1.0]), prediction=1.0), Row(article_feature=DenseVector([18.0, 2.6818, 1.3421, 1.1829, 1.129, 1.1288, 1.0202, 1.0036, 0.8665, 0.7463, 0.7246, 0.1302, 0.003, -0.0547, 0.1049, -0.0059, 0.0243, -0.0337, 0.0512, -0.0798, -0.018, -0.2112, 0.0268, 0.1038, 0.1229, 0.0568, 0.0144, 0.0542, 0.0777, 0.0451, 0.1162, -0.0914, 0.0837, -0.0483, 0.0305, -0.0651, -0.0507, 0.0758, 0.0069, 0.1505, -0.2099, -0.091, -0.0666, 0.0966, 0.1015, -0.0917, 0.0904, -0.0832, -0.0624, 0.0787, 0.0827, 0.162, -0.1955, 0.0109, 0.056, -0.04, 0.0539, 0.2001, 0.0217, -0.1235, 0.0336, 0.0519, -0.0459, 0.0479, 0.0703, 0.0315, 0.0453, 0.0066, -0.0795, 0.1192, 0.073, 0.1304, -0.1298, 0.0622, -0.1909, 0.0744, 0.0576, 0.0105, -0.006, 0.0765, -0.042, -0.0325, -0.0422, -0.0587, -0.0657, 0.0492, 0.0866, 0.0549, -0.1362, 0.0482, -0.0879, 0.0959, -0.1067, 0.0756, 0.0939, -0.0632, -0.0105, 0.1093, 0.0527, -0.12, 0.1339, 0.0532, 0.1466, 0.088, 0.0305, 0.0913, 0.0266, -0.095, -0.0952, 0.0657, 0.2223]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=1, features=DenseVector([18.0, 2.6818, 1.3421, 1.1829, 1.129, 1.1288, 1.0202, 1.0036, 0.8665, 0.7463, 0.7246, 0.1302, 0.003, -0.0547, 0.1049, -0.0059, 0.0243, -0.0337, 0.0512, -0.0798, -0.018, -0.2112, 0.0268, 0.1038, 0.1229, 0.0568, 0.0144, 0.0542, 0.0777, 0.0451, 0.1162, -0.0914, 0.0837, -0.0483, 0.0305, -0.0651, -0.0507, 0.0758, 0.0069, 0.1505, -0.2099, -0.091, -0.0666, 0.0966, 0.1015, -0.0917, 0.0904, -0.0832, -0.0624, 0.0787, 0.0827, 0.162, -0.1955, 0.0109, 0.056, -0.04, 0.0539, 0.2001, 0.0217, -0.1235, 0.0336, 0.0519, -0.0459, 0.0479, 0.0703, 0.0315, 0.0453, 0.0066, -0.0795, 0.1192, 0.073, 0.1304, -0.1298, 0.0622, -0.1909, 0.0744, 0.0576, 0.0105, -0.006, 0.0765, -0.042, -0.0325, -0.0422, -0.0587, -0.0657, 0.0492, 0.0866, 0.0549, -0.1362, 0.0482, -0.0879, 0.0959, -0.1067, 0.0756, 0.0939, -0.0632, -0.0105, 0.1093, 0.0527, -0.12, 0.1339, 0.0532, 0.1466, 0.088, 0.0305, 0.0913, 0.0266, -0.095, -0.0952, 0.0657, 0.2223, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([-54.0704, 54.0704]), probability=DenseVector([0.0, 1.0]), prediction=1.0), Row(article_feature=DenseVector([18.0, 2.5148, 1.1641, 0.6761, 0.6651, 0.3237, 0.2768, 0.2668, 0.2533, 0.2031, 0.1981, 0.0833, 0.0514, 0.031, 0.0953, -0.0219, 0.0345, 0.0106, 0.0865, -0.0843, -0.036, -0.0998, 0.0675, 0.0662, 0.0985, -0.0627, -0.0032, -0.0253, 0.005, 0.0119, 0.0266, -0.0499, 0.0298, -0.0179, -0.0036, -0.0671, -0.0667, 0.0479, 0.0902, 0.0461, -0.0992, 0.001, 0.0467, 0.1242, 0.0894, -0.0568, 0.0228, -0.0511, -0.0625, 0.0656, -0.0089, 0.0889, -0.045, -0.0421, 0.0295, -0.0094, 0.0492, 0.0872, 0.0767, -0.0641, 0.0247, -0.0354, -0.0426, 0.0896, 0.0655, 0.0106, 0.035, 0.016, -0.0308, -0.0011, 0.0547, 0.0452, -0.0987, 0.0003, -0.1365, -0.0163, 0.0752, -0.0551, -0.008, 0.0433, -0.0299, 0.0238, -0.0005, -0.0339, -0.0691, 0.0667, 0.0335, 0.0649, -0.0158, 0.0871, 0.0142, 0.0867, -0.058, 0.0213, -0.0163, 0.0038, 0.0381, 0.0216, -0.0062, -0.0101, 0.1043, 0.0566, 0.1092, 0.1094, 0.029, 0.0181, -0.0172, -0.034, -0.0687, 0.0296, 0.0949]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=1, features=DenseVector([18.0, 2.5148, 1.1641, 0.6761, 0.6651, 0.3237, 0.2768, 0.2668, 0.2533, 0.2031, 0.1981, 0.0833, 0.0514, 0.031, 0.0953, -0.0219, 0.0345, 0.0106, 0.0865, -0.0843, -0.036, -0.0998, 0.0675, 0.0662, 0.0985, -0.0627, -0.0032, -0.0253, 0.005, 0.0119, 0.0266, -0.0499, 0.0298, -0.0179, -0.0036, -0.0671, -0.0667, 0.0479, 0.0902, 0.0461, -0.0992, 0.001, 0.0467, 0.1242, 0.0894, -0.0568, 0.0228, -0.0511, -0.0625, 0.0656, -0.0089, 0.0889, -0.045, -0.0421, 0.0295, -0.0094, 0.0492, 0.0872, 0.0767, -0.0641, 0.0247, -0.0354, -0.0426, 0.0896, 0.0655, 0.0106, 0.035, 0.016, -0.0308, -0.0011, 0.0547, 0.0452, -0.0987, 0.0003, -0.1365, -0.0163, 0.0752, -0.0551, -0.008, 0.0433, -0.0299, 0.0238, -0.0005, -0.0339, -0.0691, 0.0667, 0.0335, 0.0649, -0.0158, 0.0871, 0.0142, 0.0867, -0.058, 0.0213, -0.0163, 0.0038, 0.0381, 0.0216, -0.0062, -0.0101, 0.1043, 0.0566, 0.1092, 0.1094, 0.029, 0.0181, -0.0172, -0.034, -0.0687, 0.0296, 0.0949, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([-21.1651, 21.1651]), probability=DenseVector([0.0, 1.0]), prediction=1.0), Row(article_feature=DenseVector([18.0, 2.6265, 1.0184, 0.6651, 0.3576, 0.2814, 0.2605, 0.2529, 0.2393, 0.2194, 0.2152, 0.0957, 0.0478, 0.0049, 0.1245, -0.0151, 0.0584, -0.0217, 0.0783, -0.0928, -0.0283, -0.1061, 0.049, 0.0443, 0.1149, -0.0302, 0.0287, -0.0345, 0.0156, 0.0102, 0.0218, -0.0314, 0.0371, -0.0143, 0.0293, -0.098, -0.0741, 0.0785, 0.0825, 0.0269, -0.0963, -0.0039, 0.0615, 0.1305, 0.1013, -0.0705, 0.0354, -0.064, -0.0569, 0.0663, 0.0127, 0.1048, -0.0857, -0.0323, 0.0561, -0.0038, 0.0432, 0.1249, 0.0524, -0.0524, 0.0272, -0.0364, -0.0569, 0.0943, 0.082, 0.0158, 0.0551, 0.0001, -0.0262, 0.0038, 0.0625, 0.0404, -0.0998, -0.0052, -0.1272, -0.0031, 0.0891, -0.0282, -0.0303, 0.05, -0.0466, 0.0069, -0.026, -0.0054, -0.0661, 0.0509, 0.0396, 0.0717, 0.0058, 0.0785, -0.0136, 0.0775, -0.066, 0.0343, 0.0153, 0.024, 0.0201, -0.01, -0.0245, -0.0092, 0.1099, 0.0501, 0.1, 0.1071, 0.0391, 0.0464, -0.0241, -0.0563, -0.0969, -0.0217, 0.1126]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=0, features=DenseVector([18.0, 2.6265, 1.0184, 0.6651, 0.3576, 0.2814, 0.2605, 0.2529, 0.2393, 0.2194, 0.2152, 0.0957, 0.0478, 0.0049, 0.1245, -0.0151, 0.0584, -0.0217, 0.0783, -0.0928, -0.0283, -0.1061, 0.049, 0.0443, 0.1149, -0.0302, 0.0287, -0.0345, 0.0156, 0.0102, 0.0218, -0.0314, 0.0371, -0.0143, 0.0293, -0.098, -0.0741, 0.0785, 0.0825, 0.0269, -0.0963, -0.0039, 0.0615, 0.1305, 0.1013, -0.0705, 0.0354, -0.064, -0.0569, 0.0663, 0.0127, 0.1048, -0.0857, -0.0323, 0.0561, -0.0038, 0.0432, 0.1249, 0.0524, -0.0524, 0.0272, -0.0364, -0.0569, 0.0943, 0.082, 0.0158, 0.0551, 0.0001, -0.0262, 0.0038, 0.0625, 0.0404, -0.0998, -0.0052, -0.1272, -0.0031, 0.0891, -0.0282, -0.0303, 0.05, -0.0466, 0.0069, -0.026, -0.0054, -0.0661, 0.0509, 0.0396, 0.0717, 0.0058, 0.0785, -0.0136, 0.0775, -0.066, 0.0343, 0.0153, 0.024, 0.0201, -0.01, -0.0245, -0.0092, 0.1099, 0.0501, 0.1, 0.1071, 0.0391, 0.0464, -0.0241, -0.0563, -0.0969, -0.0217, 0.1126, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([32.1088, -32.1088]), probability=DenseVector([1.0, 0.0]), prediction=0.0), Row(article_feature=DenseVector([18.0, 2.6654, 1.1789, 0.6651, 0.399, 0.3989, 0.3552, 0.3151, 0.2425, 0.239, 0.2311, 0.0625, 0.0433, -0.0027, 0.1095, -0.003, 0.0276, -0.0258, 0.0784, -0.0942, -0.0214, -0.0884, 0.0328, 0.0601, 0.101, -0.0354, 0.0209, -0.0315, 0.0051, -0.0042, 0.0148, -0.0186, 0.0396, -0.0112, 0.0159, -0.0781, -0.0825, 0.0617, 0.0345, 0.0235, -0.0952, -0.0172, 0.0679, 0.1196, 0.0852, -0.0418, 0.037, -0.08, -0.0662, 0.0304, 0.0179, 0.0881, -0.0685, -0.0516, 0.0132, 0.0178, 0.0129, 0.095, 0.075, -0.0723, 0.0331, -0.0464, -0.013, 0.0557, 0.0697, 0.0181, 0.0385, 0.0134, -0.0221, 0.0188, 0.032, 0.046, -0.0779, -0.015, -0.1209, -0.033, 0.091, -0.0331, -0.0382, 0.0388, -0.0426, -0.0269, -0.0427, -0.0317, -0.0346, 0.0714, 0.0266, 0.0501, -0.0065, 0.0882, -0.0069, 0.078, -0.0623, 0.0387, -0.0004, -0.0009, 0.0291, -0.0221, -0.0269, -0.017, 0.0745, 0.0327, 0.1106, 0.0912, 0.0102, 0.0094, -0.0546, -0.0413, -0.0614, 0.0174, 0.0657]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=1, features=DenseVector([18.0, 2.6654, 1.1789, 0.6651, 0.399, 0.3989, 0.3552, 0.3151, 0.2425, 0.239, 0.2311, 0.0625, 0.0433, -0.0027, 0.1095, -0.003, 0.0276, -0.0258, 0.0784, -0.0942, -0.0214, -0.0884, 0.0328, 0.0601, 0.101, -0.0354, 0.0209, -0.0315, 0.0051, -0.0042, 0.0148, -0.0186, 0.0396, -0.0112, 0.0159, -0.0781, -0.0825, 0.0617, 0.0345, 0.0235, -0.0952, -0.0172, 0.0679, 0.1196, 0.0852, -0.0418, 0.037, -0.08, -0.0662, 0.0304, 0.0179, 0.0881, -0.0685, -0.0516, 0.0132, 0.0178, 0.0129, 0.095, 0.075, -0.0723, 0.0331, -0.0464, -0.013, 0.0557, 0.0697, 0.0181, 0.0385, 0.0134, -0.0221, 0.0188, 0.032, 0.046, -0.0779, -0.015, -0.1209, -0.033, 0.091, -0.0331, -0.0382, 0.0388, -0.0426, -0.0269, -0.0427, -0.0317, -0.0346, 0.0714, 0.0266, 0.0501, -0.0065, 0.0882, -0.0069, 0.078, -0.0623, 0.0387, -0.0004, -0.0009, 0.0291, -0.0221, -0.0269, -0.017, 0.0745, 0.0327, 0.1106, 0.0912, 0.0102, 0.0094, -0.0546, -0.0413, -0.0614, 0.0174, 0.0657, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([-21.1104, 21.1104]), probability=DenseVector([0.0, 1.0]), prediction=1.0), Row(article_feature=DenseVector([18.0, 2.6818, 2.4223, 1.4873, 1.2856, 1.1535, 1.1389, 1.1155, 0.8235, 0.7417, 0.7324, 0.1488, 0.1157, 0.0939, 0.2115, -0.0271, 0.0357, -0.1109, 0.1074, -0.1248, -0.1019, -0.2566, 0.0641, 0.0761, 0.2292, -0.0063, 0.0104, -0.0001, 0.0809, 0.0549, 0.0554, -0.1041, 0.1419, -0.034, 0.0583, -0.1595, -0.0988, 0.1436, 0.1244, 0.0288, -0.277, -0.1094, 0.0728, 0.2335, 0.2044, -0.1232, 0.0362, -0.0676, -0.1165, 0.143, 0.0562, 0.1929, -0.1697, -0.0649, 0.1593, 0.0121, -0.0625, 0.2219, -0.0103, -0.1582, 0.0708, -0.0517, -0.0974, 0.1914, 0.0465, 0.0045, 0.0991, -0.0894, -0.0944, 0.1545, 0.0678, 0.1513, -0.206, 0.0176, -0.3182, 0.0232, 0.1828, 0.0267, -0.0538, 0.0997, -0.0754, -0.0044, 0.0199, -0.0256, -0.1297, 0.1187, 0.0881, 0.1341, -0.0302, 0.1794, -0.1543, 0.1252, -0.1617, -0.0523, 0.0793, -0.0431, 0.0822, -0.0603, 0.0747, -0.0701, 0.2572, 0.1189, 0.2234, 0.2088, 0.0231, 0.0506, -0.01, -0.1732, -0.2522, -0.0789, 0.3554]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=0, features=DenseVector([18.0, 2.6818, 2.4223, 1.4873, 1.2856, 1.1535, 1.1389, 1.1155, 0.8235, 0.7417, 0.7324, 0.1488, 0.1157, 0.0939, 0.2115, -0.0271, 0.0357, -0.1109, 0.1074, -0.1248, -0.1019, -0.2566, 0.0641, 0.0761, 0.2292, -0.0063, 0.0104, -0.0001, 0.0809, 0.0549, 0.0554, -0.1041, 0.1419, -0.034, 0.0583, -0.1595, -0.0988, 0.1436, 0.1244, 0.0288, -0.277, -0.1094, 0.0728, 0.2335, 0.2044, -0.1232, 0.0362, -0.0676, -0.1165, 0.143, 0.0562, 0.1929, -0.1697, -0.0649, 0.1593, 0.0121, -0.0625, 0.2219, -0.0103, -0.1582, 0.0708, -0.0517, -0.0974, 0.1914, 0.0465, 0.0045, 0.0991, -0.0894, -0.0944, 0.1545, 0.0678, 0.1513, -0.206, 0.0176, -0.3182, 0.0232, 0.1828, 0.0267, -0.0538, 0.0997, -0.0754, -0.0044, 0.0199, -0.0256, -0.1297, 0.1187, 0.0881, 0.1341, -0.0302, 0.1794, -0.1543, 0.1252, -0.1617, -0.0523, 0.0793, -0.0431, 0.0822, -0.0603, 0.0747, -0.0701, 0.2572, 0.1189, 0.2234, 0.2088, 0.0231, 0.0506, -0.01, -0.1732, -0.2522, -0.0789, 0.3554, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([96.6503, -96.6503]), probability=DenseVector([1.0, 0.0]), prediction=0.0), Row(article_feature=DenseVector([18.0, 2.2568, 1.312, 1.2308, 1.0965, 1.0449, 0.9641, 0.8947, 0.7438, 0.7137, 0.6651, 0.0795, 0.0376, -0.0171, 0.0841, 0.0552, 0.0159, -0.0698, 0.0305, -0.08, -0.0773, -0.095, 0.0752, 0.1102, 0.1526, 0.0353, -0.0241, -0.0102, 0.057, 0.0644, 0.1255, -0.0912, 0.0196, 0.0156, 0.0278, -0.1028, 0.0624, 0.0151, 0.0044, 0.0374, -0.212, -0.0633, 0.0293, 0.1679, 0.1544, -0.1015, 0.073, -0.0512, -0.0817, 0.0991, 0.0032, 0.0923, -0.0688, -0.0211, 0.1541, -0.0245, -0.035, 0.1239, 0.0524, -0.0933, 0.0949, -0.0027, 0.0086, 0.064, 0.1451, 0.0232, 0.1071, -0.0365, -0.0363, 0.0657, 0.0473, 0.0885, -0.1571, 0.0828, -0.1818, -0.0012, 0.0756, -0.0093, -0.0437, 0.0229, 0.0621, -0.0501, 0.0191, 0.0513, -0.0623, 0.0108, 0.0943, 0.0417, -0.1148, 0.0893, -0.0291, 0.1158, -0.1257, 0.0215, 0.1155, -0.0297, 0.0386, 0.021, 0.0534, -0.0299, 0.1745, -0.0065, 0.0929, 0.1305, 0.0404, 0.0426, -0.0928, -0.1047, -0.0975, 0.0162, 0.1468]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=1, features=DenseVector([18.0, 2.2568, 1.312, 1.2308, 1.0965, 1.0449, 0.9641, 0.8947, 0.7438, 0.7137, 0.6651, 0.0795, 0.0376, -0.0171, 0.0841, 0.0552, 0.0159, -0.0698, 0.0305, -0.08, -0.0773, -0.095, 0.0752, 0.1102, 0.1526, 0.0353, -0.0241, -0.0102, 0.057, 0.0644, 0.1255, -0.0912, 0.0196, 0.0156, 0.0278, -0.1028, 0.0624, 0.0151, 0.0044, 0.0374, -0.212, -0.0633, 0.0293, 0.1679, 0.1544, -0.1015, 0.073, -0.0512, -0.0817, 0.0991, 0.0032, 0.0923, -0.0688, -0.0211, 0.1541, -0.0245, -0.035, 0.1239, 0.0524, -0.0933, 0.0949, -0.0027, 0.0086, 0.064, 0.1451, 0.0232, 0.1071, -0.0365, -0.0363, 0.0657, 0.0473, 0.0885, -0.1571, 0.0828, -0.1818, -0.0012, 0.0756, -0.0093, -0.0437, 0.0229, 0.0621, -0.0501, 0.0191, 0.0513, -0.0623, 0.0108, 0.0943, 0.0417, -0.1148, 0.0893, -0.0291, 0.1158, -0.1257, 0.0215, 0.1155, -0.0297, 0.0386, 0.021, 0.0534, -0.0299, 0.1745, -0.0065, 0.0929, 0.1305, 0.0404, 0.0426, -0.0928, -0.1047, -0.0975, 0.0162, 0.1468, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([-38.9233, 38.9233]), probability=DenseVector([0.0, 1.0]), prediction=1.0), Row(article_feature=DenseVector([18.0, 2.6818, 1.1932, 0.6125, 0.5196, 0.4166, 0.3489, 0.2493, 0.2439, 0.2373, 0.2229, 0.0859, 0.0367, 0.0087, 0.1203, -0.0331, 0.0277, 0.001, 0.0864, -0.0847, -0.0329, -0.1187, 0.0509, 0.0372, 0.1115, -0.0379, 0.0381, -0.0451, 0.0102, 0.0013, 0.0086, -0.0071, 0.0363, -0.0249, 0.011, -0.0884, -0.0796, 0.0668, 0.0763, 0.0367, -0.0907, -0.0062, 0.0544, 0.1251, 0.0976, -0.0437, 0.0196, -0.0576, -0.078, 0.0597, -0.0104, 0.1231, -0.0949, -0.0121, 0.0275, 0.0015, 0.023, 0.1064, 0.0576, -0.0727, 0.0329, -0.0411, -0.0419, 0.0667, 0.1159, 0.0083, 0.0501, -0.001, -0.0286, 0.0094, 0.0704, 0.0397, -0.0887, 0.0237, -0.1286, -0.0232, 0.0693, -0.0338, -0.016, 0.0486, -0.0506, -0.0178, -0.0298, -0.0026, -0.063, 0.0643, 0.0464, 0.0436, -0.0166, 0.0943, 0.0052, 0.0786, -0.0682, 0.0301, 0.0203, 0.014, 0.0357, -0.0052, -0.0433, 0.0117, 0.0846, 0.0526, 0.0968, 0.1111, 0.0231, 0.036, -0.0337, -0.04, -0.0977, 0.0099, 0.1223]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=0, features=DenseVector([18.0, 2.6818, 1.1932, 0.6125, 0.5196, 0.4166, 0.3489, 0.2493, 0.2439, 0.2373, 0.2229, 0.0859, 0.0367, 0.0087, 0.1203, -0.0331, 0.0277, 0.001, 0.0864, -0.0847, -0.0329, -0.1187, 0.0509, 0.0372, 0.1115, -0.0379, 0.0381, -0.0451, 0.0102, 0.0013, 0.0086, -0.0071, 0.0363, -0.0249, 0.011, -0.0884, -0.0796, 0.0668, 0.0763, 0.0367, -0.0907, -0.0062, 0.0544, 0.1251, 0.0976, -0.0437, 0.0196, -0.0576, -0.078, 0.0597, -0.0104, 0.1231, -0.0949, -0.0121, 0.0275, 0.0015, 0.023, 0.1064, 0.0576, -0.0727, 0.0329, -0.0411, -0.0419, 0.0667, 0.1159, 0.0083, 0.0501, -0.001, -0.0286, 0.0094, 0.0704, 0.0397, -0.0887, 0.0237, -0.1286, -0.0232, 0.0693, -0.0338, -0.016, 0.0486, -0.0506, -0.0178, -0.0298, -0.0026, -0.063, 0.0643, 0.0464, 0.0436, -0.0166, 0.0943, 0.0052, 0.0786, -0.0682, 0.0301, 0.0203, 0.014, 0.0357, -0.0052, -0.0433, 0.0117, 0.0846, 0.0526, 0.0968, 0.1111, 0.0231, 0.036, -0.0337, -0.04, -0.0977, 0.0099, 0.1223, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([23.429, -23.429]), probability=DenseVector([1.0, 0.0]), prediction=0.0), Row(article_feature=DenseVector([18.0, 2.6818, 1.2738, 0.7039, 0.6205, 0.4325, 0.3708, 0.3568, 0.2825, 0.2664, 0.264, 0.1209, 0.0499, 0.0233, 0.1055, -0.0531, 0.0331, -0.017, 0.0956, -0.1122, -0.0273, -0.1311, 0.0559, 0.0301, 0.127, -0.0421, 0.0467, -0.003, 0.0118, 0.0066, 0.0252, -0.0458, 0.0414, -0.0355, 0.009, -0.1247, -0.0846, 0.0657, 0.0915, 0.0429, -0.1134, -0.0164, 0.0529, 0.1239, 0.0908, -0.0864, 0.0422, -0.0727, -0.079, 0.077, 0.0007, 0.122, -0.1252, -0.0176, 0.0412, 0.0043, 0.0604, 0.1229, 0.0491, -0.0894, 0.0495, -0.0345, -0.0514, 0.1018, 0.108, 0.0289, 0.0564, -0.0004, -0.042, 0.0309, 0.0683, 0.0528, -0.0778, 0.0234, -0.1457, -0.0124, 0.0821, -0.0337, -0.0138, 0.0539, -0.0726, 0.0095, -0.0327, -0.0168, -0.0805, 0.0573, 0.0561, 0.0497, -0.013, 0.0837, -0.015, 0.0643, -0.0913, 0.0389, 0.0169, 0.0177, 0.0331, 0.0249, -0.0073, -0.0185, 0.1299, 0.0552, 0.102, 0.1204, 0.0089, 0.0442, -0.0048, -0.0582, -0.121, -0.0039, 0.177]), user_feature=DenseVector([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), clicked=0, features=DenseVector([18.0, 2.6818, 1.2738, 0.7039, 0.6205, 0.4325, 0.3708, 0.3568, 0.2825, 0.2664, 0.264, 0.1209, 0.0499, 0.0233, 0.1055, -0.0531, 0.0331, -0.017, 0.0956, -0.1122, -0.0273, -0.1311, 0.0559, 0.0301, 0.127, -0.0421, 0.0467, -0.003, 0.0118, 0.0066, 0.0252, -0.0458, 0.0414, -0.0355, 0.009, -0.1247, -0.0846, 0.0657, 0.0915, 0.0429, -0.1134, -0.0164, 0.0529, 0.1239, 0.0908, -0.0864, 0.0422, -0.0727, -0.079, 0.077, 0.0007, 0.122, -0.1252, -0.0176, 0.0412, 0.0043, 0.0604, 0.1229, 0.0491, -0.0894, 0.0495, -0.0345, -0.0514, 0.1018, 0.108, 0.0289, 0.0564, -0.0004, -0.042, 0.0309, 0.0683, 0.0528, -0.0778, 0.0234, -0.1457, -0.0124, 0.0821, -0.0337, -0.0138, 0.0539, -0.0726, 0.0095, -0.0327, -0.0168, -0.0805, 0.0573, 0.0561, 0.0497, -0.013, 0.0837, -0.015, 0.0643, -0.0913, 0.0389, 0.0169, 0.0177, 0.0331, 0.0249, -0.0073, -0.0185, 0.1299, 0.0552, 0.102, 0.1204, 0.0089, 0.0442, -0.0048, -0.0582, -0.121, -0.0039, 0.177, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), rawPrediction=DenseVector([27.1939, -27.1939]), probability=DenseVector([1.0, 0.0]), prediction=0.0)]
# 四、评估部分
def get_ctr(row):
return float(row.clicked), float(row.probability[1])
score_label = sort_res.select(["clicked", "probability"]).rdd.map(get_ctr)
print("预测结果对比", score_label.take(10))
# [(0.0, 8.066185650702151e-13), (1.0, 0.999999999998074), (1.0, 1.0), (1.0, 0.9999999993571316), (0.0, 1.135851844955037e-14), (1.0, 0.9999999993210209), (0.0, 1.0599841392343561e-42), (1.0, 1.0), (0.0, 6.68237764147462e-11), (0.0, 1.5482022140641281e-12)]
# 评估
from sklearn.metrics import accuracy_score, roc_auc_score
import numpy as np
arr = np.array(score_label.collect())
# AUC
print(roc_auc_score(arr[:, 0], arr[:, 1])) # 0.990740740741
# 准确率
print(accuracy_score(arr[:, 0], arr[:, 1].round())) # 0.933333333333