1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# Password for the 'elastic' user (at least 6 characters)
ELASTIC_PASSWORD=123456

# Password for the 'kibana_system' user (at least 6 characters)
KIBANA_PASSWORD=123456

# Version of Elastic products
STACK_VERSION=9.2.3

# Set the cluster name
CLUSTER_NAME=elasticsearch-cluster

# Set to 'basic' or 'trial' to automatically start the 30-day trial
LICENSE=basic

# Port to expose Elasticsearch HTTP API to the host
ES_PORT=9200

# Port to expose Kibana to the host
KIBANA_PORT=5601

# Increase or decrease based on the available host memory (in bytes)
MEM_LIMIT=8589934592

# Project namespace (defaults to the current folder name if not set)
COMPOSE_PROJECT_NAME=elasticsearch-project

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

import gzip
import json
import os
from datetime import datetime
from urllib.parse import urlparse

import requests
from elasticsearch import Elasticsearch, helpers

ES_URL = "https://localhost:9200"
ES_USER = "elastic"
ES_PASSWORD = "123456"
INDEX_NAME = "eo_logs"
DOWNLOAD_DIR = "downloaded_logs"

es = Elasticsearch([ES_URL], basic_auth=(ES_USER, ES_PASSWORD), verify_certs=False, ssl_show_warn=False)
os.makedirs(DOWNLOAD_DIR, exist_ok=True)


def download_file(url):
    filename = os.path.basename(urlparse(url).path)
    filepath = os.path.join(DOWNLOAD_DIR, filename)
    if os.path.exists(filepath):
        print(f"文件已存在: {filename}")
        return filepath
    print(f"下载: {filename}")
    response = requests.get(url, stream=True, timeout=300)
    with open(filepath, 'wb') as f:
        for chunk in response.iter_content(chunk_size=8192):
            if chunk:
                f.write(chunk)
    return filepath


def parse_gz(filepath):
    logs = []
    print(f"解析: {os.path.basename(filepath)}")
    with gzip.open(filepath, 'rt', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if line:
                log = json.loads(line)
                log['_source_file'] = os.path.basename(filepath)
                log['_import_time'] = datetime.utcnow().isoformat()
                logs.append(log)

    print(f"解析完成: {len(logs)} 条")
    return logs


def save_to_es(logs):
    if not logs:
        return
    print(f"保存 {len(logs)} 条到 ES")
    actions = [{"_index": INDEX_NAME, "_source": log} for log in logs]
    success, _ = helpers.bulk(es, actions, chunk_size=1000, request_timeout=60)
    print(f"保存完成: {success} 条")


def process_url(url):
    filepath = download_file(url)
    logs = parse_gz(filepath)
    save_to_es(logs)


def main():
    with open("urls.txt", 'r') as f:
        urls = [line.strip() for line in f if line.strip()]
    print(f"开始处理 {len(urls)} 个文件\n")
    for i, url in enumerate(urls, 1):
        print(f"\n[{i}/{len(urls)}]")
        process_url(url)
    print("\n处理完成!")


if __name__ == "__main__":
    main()

1
2

~ curl 'https://127.0.0.1:9200/eo_logs/_count' --header 'Authorization: Basic ZWxhc3RpYzo9dk5Cc0QwSTNZRWFPa2RoZFFhZg==' -k
{"count":31398691,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0}}%

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238

{
  "eo_logs": {
    "aliases": {},
    "mappings": {
      "properties": {
        "ClientIP": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ClientISP": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ClientRegion": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ClientState": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ContentID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "EdgeCacheStatus": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "EdgeFunctionSubrequest": {
          "type": "long"
        },
        "EdgeInternalTime": {
          "type": "long"
        },
        "EdgeResponseBodyBytes": {
          "type": "long"
        },
        "EdgeResponseBytes": {
          "type": "long"
        },
        "EdgeResponseStatusCode": {
          "type": "long"
        },
        "EdgeResponseTime": {
          "type": "long"
        },
        "EdgeServerID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "EdgeServerIP": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "ParentRequestID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RemotePort": {
          "type": "long"
        },
        "RequestBytes": {
          "type": "long"
        },
        "RequestHost": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestID": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestMethod": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestProtocol": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestRange": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestReferer": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestStatus": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestTime": {
          "type": "date"
        },
        "RequestUA": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestUrl": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "RequestUrlQueryString": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        },
        "_import_time": {
          "type": "date"
        },
        "_source_file": {
          "type": "text",
          "fields": {
            "keyword": {
              "type": "keyword",
              "ignore_above": 256
            }
          }
        }
      }
    },
    "settings": {
      "index": {
        "routing": {
          "allocation": {
            "include": {
              "_tier_preference": "data_content"
            }
          }
        },
        "number_of_shards": "1",
        "provided_name": "eo_logs",
        "creation_date": "1766816305347",
        "number_of_replicas": "1",
        "uuid": "wi9l88cjRh-Kq7lgl4NReg",
        "version": {
          "created": "9039003"
        }
      }
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

POST /eo_logs/_search
{
    "size": 0,
    "query": {
        "bool": {
            "filter": [
                {
                    "term": {
                        "RequestHost.keyword": "static.example.com"
                    }
                }
            ]
        }
    },
    "aggs": {
        "edge_response_stats": {
            "stats": {
                "field": "EdgeResponseTime"
            }
        },
        "edge_response_percentiles": {
            "percentiles": {
                "field": "EdgeResponseTime",
                "percents": [
                    50,
                    90,
                    95,
                    99
                ]
            }
        },
        "edge_response_hist": {
            "histogram": {
                "field": "EdgeResponseTime",
                "interval": 50,
                "min_doc_count": 1
            }
        }
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

{
  "took": 3128,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 10000,
      "relation": "gte"
    },
    "max_score": null,
    "hits": []
  },
  "aggregations": {
    "edge_response_percentiles": {
      "values": {
        "50.0": 5.014287434842656,
        "90.0": 25.778307762642324,
        "95.0": 73.78316545752277,
        "99.0": 593.9728031414846
      }
    },
    "edge_response_hist": {
      "buckets": [
        {
          "key": 0.0,
          "doc_count": 25997272
        },
        {
          "key": 50.0,
          "doc_count": 841843
        },
        {
          "key": 100.0,
          "doc_count": 377168
        },
        {
          "key": 150.0,
          "doc_count": 109181
        },
        {
          "key": 200.0,
          "doc_count": 53672
        },
        {
          "key": 250.0,
          "doc_count": 37425
        },
        {
          "key": 300.0,
          "doc_count": 32744
        },
        {
          "key": 350.0,
          "doc_count": 36445
        },
        {
          "key": 400.0,
          "doc_count": 26137
        },
        {
          "key": 450.0,
          "doc_count": 22807
        },
        {
          "key": 500.0,
          "doc_count": 21111
        },
        {
          "key": 550.0,
          "doc_count": 16784
        },
        {
          "key": 600.0,
          "doc_count": 13214
        },
        {
          "key": 650.0,
          "doc_count": 11211
        },
        {
          "key": 700.0,
          "doc_count": 11760
        },
        {
          "key": 750.0,
          "doc_count": 11911
        },
        {
          "key": 800.0,
          "doc_count": 10381
        },
        {
          "key": 850.0,
          "doc_count": 9158
        },
        {
          "key": 900.0,
          "doc_count": 6851
        },
        {
          "key": 950.0,
          "doc_count": 5822
        },
        {
          "key": 1000.0,
          "doc_count": 5195
        },
        ...
      ]
    },
    "edge_response_stats": {
      "count": 27840645,
      "min": 1.0,
      "max": 707706.0,
      "avg": 46.91420216737076,
      "sum": 1.306121648E9
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

POST /eo_logs/_search
{
    "size": 0,
    "query": {
        "bool": {
            "filter": [
                {
                    "term": {
                        "RequestHost.keyword": "static.example.com"
                    }
                },
                {
                    "exists": {
                        "field": "RequestUrl.keyword"
                    }
                },
                {
                    "exists": {
                        "field": "EdgeResponseTime"
                    }
                }
            ]
        }
    },
    "aggs": {
        "by_url": {
            "terms": {
                "field": "RequestUrl.keyword",
                "size": 200,
                "order": {
                    "p95_edge_response[95.0]": "desc"
                }
            },
            "aggs": {
                "p95_edge_response": {
                    "percentiles": {
                        "field": "EdgeResponseTime",
                        "percents": [
                            95
                        ]
                    }
                },
                "avg_edge_response": {
                    "avg": {
                        "field": "EdgeResponseTime"
                    }
                },
                "count_requests": {
                    "value_count": {
                        "field": "EdgeResponseTime"
                    }
                }
            }
        }
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

POST /eo_logs/_search
{
    "size": 0,
    "query": {
        "bool": {
            "filter": [
                {
                    "term": {
                        "RequestHost.keyword": "static.example.com"
                    }
                },
                {
                    "exists": {
                        "field": "RequestUrl.keyword"
                    }
                },
                {
                    "exists": {
                        "field": "EdgeResponseTime"
                    }
                },
                {
                    "exists": {
                        "field": "EdgeResponseBodyBytes"
                    }
                },
                {
                    "range": {
                        "EdgeResponseBodyBytes": {
                            "gt": 0
                        }
                    }
                },
                {
                    "range": {
                        "EdgeResponseTime": {
                            "gt": 0
                        }
                    }
                }
            ]
        }
    },
    "aggs": {
        "by_url": {
            "terms": {
                "field": "RequestUrl.keyword",
                "size": 2000,
                "order": {
                    "avg_kbps": "asc"
                }
            },
            "aggs": {
                "avg_kbps": {
                    "avg": {
                        "script": {
                            "lang": "painless",
                            "source": "double b = doc['EdgeResponseBodyBytes'].value; double t = doc['EdgeResponseTime'].value; return (b / t) * (1000.0 / 1024.0);"
                        }
                    }
                },
                "p95_kbps": {
                    "percentiles": {
                        "script": {
                            "lang": "painless",
                            "source": "double b = doc['EdgeResponseBodyBytes'].value; double t = doc['EdgeResponseTime'].value; return (b / t) * (1000.0 / 1024.0);"
                        },
                        "percents": [
                            95
                        ]
                    }
                },
                "avg_time_ms": {
                    "avg": {
                        "field": "EdgeResponseTime"
                    }
                },
                "avg_body_bytes": {
                    "avg": {
                        "field": "EdgeResponseBodyBytes"
                    }
                },
                "req_count": {
                    "value_count": {
                        "field": "EdgeResponseTime"
                    }
                }
            }
        }
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

server {
    listen 8965;

    # 鉴权接口，仅供 Nginx 内部 auth_request 使用
    location = /auth {
        internal;  # 该接口只能被 Nginx 内部请求，防止外部访问
        # 转发到实际的认证服务
        proxy_pass http://localhost:5001/api/auth/verify;
        # 不转发请求体，提升效率
        proxy_pass_request_body off;
        # 防止后端因 Content-Length 不确定而报错
        proxy_set_header Content-Length "";
        # 将客户端传来的 Authorization 头（JWT Token）传给认证服务
        proxy_set_header Authorization $http_authorization;
        # 传入原始请求路径，供认证服务判断路径是否需要鉴权
        proxy_set_header X-Original-URI $request_uri;
    }

    # 所有 / 路径下的请求都进行认证
    location / {
        # 认证请求会先调用上面的 /auth 接口
        auth_request /auth;
        # 如果认证失败（如返回 401），跳转到自定义处理逻辑
        error_page 401 = @unauthorized;
        # 从 /auth 的响应头中提取用户信息
        auth_request_set $user_id $upstream_http_x_user_id;
        # 把用户信息注入到请求头中，转发给后端业务服务
        proxy_set_header X-User-ID $user_id;
        # 转发到后端服务
        proxy_pass http://localhost:5001;
    }

    # 自定义未授权响应（认证失败时返回）
    location @unauthorized {
        # 返回 401 状态码 + 文本内容
        return 401 "Unauthorized";
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

import jwt
from flask import Flask, request, make_response, jsonify
from jwt import InvalidTokenError

app = Flask(__name__)

SECRET_KEY = "a-string-secret-at-least-256-bits-long"

# 路径白名单，不需要鉴权的接口
PUBLIC_PATHS = [
    "/api/public/hello",
    "/api/login",
    "/api/register"
]


@app.route("/api/auth/verify", methods=["GET"])
def verify_token():
    original_uri = request.headers.get("X-Original-URI", "")
    # 不需要鉴权的接口，直接返回200
    if original_uri in PUBLIC_PATHS:
        return "", 200

    # 否则继续验证 JWT
    auth_header = request.headers.get("Authorization")
    if not auth_header or not auth_header.startswith("Bearer "):
        return "Missing or invalid Authorization header", 401

    # 解析得到token
    token = auth_header.split(" ", 1)[1]
    try:
        payload = jwt.decode(token, SECRET_KEY, algorithms=["HS256"])
        response = make_response("", 200)
        response.headers["X-User-ID"] = str(payload.get("user_id", ""))
        return response
    except InvalidTokenError:
        return "Invalid token", 401


@app.route("/api/hello")
def hello():
    return jsonify({"message": "hello from auth", "user_id": request.headers.get("X-User-ID")})


@app.route("/api/public/hello")
def ping():
    return {"msg": "hello without auth"}


if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5001)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

return function(_, ctx)
  local core = require("apisix.core")
  local jwt = require("resty.jwt")

  local auth_header = ctx.var.http_authorization
  if not auth_header then
    return
  end

  local token = auth_header:match("Bearer%s+(.+)")
  if not token then
    return
  end

  local obj = jwt:load_jwt(token)
  if obj and obj.valid and obj.payload then
    if obj.payload.user_id then
      core.request.set_header("X-User-Id", obj.payload.user_id)
    end
    if obj.payload.role then
      core.request.set_header("X-User-Role", obj.payload.role)
    end
  end
end

1
2
3
4
5
6

{
    "key": "app-key",
    "user_id": 100001,
    "role": "admin",
    "exp": 1900000000
}

1
2
3
4
5
6
7
8
9
10
11
12
13

{
  "headers": {
    "Accept": "*/*",
    "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJrZXkiOiJhcHAta2V5IiwidXNlcl9pZCI6MTAwMDAxLCJyb2xlIjoiYWRtaW4iLCJleHAiOjE5MDAwMDAwMDB9.qG7PNPz2XlatmjrhNW_xf6SmI8T9JSIx2lJVJcAox0I",
    "Host": "httpbin.org",
    "User-Agent": "curl/7.81.0",
    "X-Amzn-Trace-Id": "Root=1-68709903-309891501348175943af3223",
    "X-Consumer-Username": "app",
    "X-Forwarded-Host": "httpbin.org",
    "X-User-Id": "100001",
    "X-User-Role": "admin"
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

def similarity(users):
    w = defaultdict(dict)
    for u, v in combinations(users.keys(), 2):
        r1 = len(users[u] & users[v])
        r2 = math.sqrt(len(users[u]) * len(users[v]) * 1.0)
        r = r1 / r2
        w[u][v], w[v][u] = r, r  # 保存两次，方便后面使用
    return w

def main():
    users = {
        'A': {'a', 'b', 'd'},
        'B': {'a', 'c'},
        'C': {'b', 'e'},
        'D': {'c', 'd', 'e'}
    }
    for k, v in similarity(users).items():
        print(f'{k}: {json.dumps(v)}')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

def recommend(user, users, w, k):
    """
    :param user: 计算指定用户的物品推荐程度
    :param users: 数据集
    :param w: 前一步计算得到的用户兴趣相似度
    :param k: 取k个兴趣最相似的用户
    :return:
    """
    rank = defaultdict(float)
    # 获取指定用户和其它用户的兴趣相似度，并按照相似度从大到小排序，取前k个数据
    for v, wuv in sorted(w[user].items(), key=lambda item: item[1], reverse=True)[:k]:
        # 取出指定用户的数据集
        for i in users[v]:
            # 如果这个数据已经在当前用户的数据集里面，跳过，因为已经感兴趣的数据不需要再次推荐
            if i in users[user]:
                continue
            rank[i] += wuv
    return rank

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

import json
import math
from collections import defaultdict
from itertools import combinations


def similarity(users):
    w = defaultdict(dict)
    for u, v in combinations(users.keys(), 2):
        r1 = len(users[u] & users[v])
        r2 = math.sqrt(len(users[u]) * len(users[v]) * 1.0)
        r = r1 / r2
        w[u][v], w[v][u] = r, r  # 保存两次，方便后面使用
    return w


def recommend(user, users, w, k):
    """
    :param user: 计算指定用户的物品推荐程度
    :param users: 数据集
    :param w: 前一步计算得到的用户兴趣相似度
    :param k: 取k个兴趣最相似的用户
    :return:
    """
    rank = defaultdict(float)
    # 获取指定用户和其它用户的兴趣相似度，并按照相似度从大到小排序，取前k个数据
    for v, wuv in sorted(w[user].items(), key=lambda item: item[1], reverse=True)[:k]:
        # 取出指定用户的数据集
        for i in users[v]:
            # 如果这个数据已经在当前用户的数据集里面，跳过，因为已经感兴趣的数据不需要再次推荐
            if i in users[user]:
                continue
            rank[i] += wuv
    return rank


def main():
    users = {
        'A': {'a', 'b', 'd'},
        'B': {'a', 'c'},
        'C': {'b', 'e'},
        'D': {'c', 'd', 'e'}
    }

    w = similarity(users)
    for k, v in w.items():
        print(f'{k}: {json.dumps(v)}')

    rank = recommend('C', users, w, 3)
    for k, v in sorted(rank.items(), key=lambda item: item[1], reverse=True):
        print(f'{k}: {v}')


if __name__ == '__main__':
    main()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

number = 'number'
def item_similarity(train):
    # c[i][number]表示使用物品i的用户数量
    # c[i][j]表示同时交互物品i和j的用户数
    c = defaultdict(lambda: defaultdict(int))
    for user, items in train.items():
        for i in items:
            # 统计每个物品被交互的总次数
            c[i][number] += 1
            # 统计物品i与其他物品的共现次数
            for j in items:
                if i == j:
                    continue
                c[i][j] += 1
    # 计算最终的相似度矩阵 w
    w = defaultdict(dict)
    for i, related_items in c.items():
        for j, cij in related_items.items():
            if j == number: continue
            # 余弦相似度公式
            similarity = cij / math.sqrt(c[i][number] * c[j][number])
            w[i][j] = similarity
    return w

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

def recommend(interacted_items: Union[set, dict], w, k):
    """
    :param interacted_items: 指定用户交互过的物品
    :param w: 物品的相似度
    :param k: 取最相似的k个物品
    :return:
    """
    if isinstance(interacted_items, set):  # 如果只有物品，没有评分，那么将评分统一设置为1
        interacted_items = {k: 1 for k in interacted_items}
    rank = defaultdict(float)
    # 用户交互过的物品，和用户对这个物品的评分
    for item, score in interacted_items.items():
        # 物品的相似度信息，得到related_item和item的相似度similarity，按照相似度的值从大到小排序，取k个值
        for related_item, similarity in sorted(w[item].items(), key=lambda x: x[1], reverse=True)[:k]:
            # 如果这个物品已经被用户交互过了，跳过
            if related_item in interacted_items:
                continue
            # 计算相关的物品的相似度评分
            rank[related_item] += score * similarity
    return rank

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

import math
from collections import defaultdict
from typing import Union

number = 'number'


def item_similarity(train):
    # c[i][number]表示使用物品i的用户数量
    # c[i][j]表示同时交互物品i和j的用户数
    c = defaultdict(lambda: defaultdict(int))
    for user, items in train.items():
        for i in items:
            # 统计每个物品被交互的总次数
            c[i][number] += 1
            # 统计物品i与其他物品的共现次数
            for j in items:
                if i == j:
                    continue
                c[i][j] += 1

    # 计算最终的相似度矩阵 w
    w = defaultdict(dict)
    for i, related_items in c.items():
        for j, cij in related_items.items():
            if j == number: continue
            # 余弦相似度公式
            similarity = cij / math.sqrt(c[i][number] * c[j][number])
            w[i][j] = similarity

    return w


def recommend(interacted_items: Union[set, dict], w, k):
    """
    :param interacted_items: 指定用户交互过的物品
    :param w: 物品的相似度
    :param k: 取最相似的k个物品
    :return:
    """
    if isinstance(interacted_items, set):  # 如果只有物品，没有评分，那么将评分统一设置为1
        interacted_items = {k: 1 for k in interacted_items}
    rank = defaultdict(float)
    # 用户交互过的物品，和用户对这个物品的评分
    for item, score in interacted_items.items():
        # 物品的相似度信息，得到related_item和item的相似度similarity，按照相似度的值从大到小排序，取k个值
        for related_item, similarity in sorted(w[item].items(), key=lambda x: x[1], reverse=True)[:k]:
            # 如果这个物品已经被用户交互过了，跳过
            if related_item in interacted_items:
                continue
            # 计算相关的物品的相似度评分
            rank[related_item] += score * similarity
    return rank


def main():
    users = {
        'A': {'a', 'b', 'd'},
        'B': {'a', 'c'},
        'C': {'b', 'e', 'a'},
        'D': {'c', 'd', 'e'}
    }
    w = item_similarity(users)
    for k, v in w.items():
        print(f'{k}: {dict(sorted(v.items(), key=lambda item: item[1], reverse=True))}')

    rank = recommend(users['B'], w, 3)
    for k, v in sorted(rank.items(), key=lambda item: item[1], reverse=True):
        print(f'{k}: {v}')


if __name__ == '__main__':
    main()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

def precision_recall(test_data, train_data, n, recommend_func):
    """
    计算推荐系统在测试集上的准确率和召回率

    :param test_data: dict，用户 -> 测试集中真实交互物品列表
    :param train_data: dict，用户 -> 训练集中交互物品列表（用于生成推荐）
    :param n: int，每个用户推荐的物品数量
    :param recommend_func: function(user, n, train_data)，返回推荐物品列表
    :return: [recall, precision]
    """
    hit = 0  # 交集
    total_actual = 0  # 所有用户的真实物品总数
    total_recommend = 0  # 所有推荐物品总数

    for user, actual_items in test_data.items():
        # 计算推荐物品
        recommended_items = recommend_func(user, n, train_data)
        # 计算交集
        hit += len(set(recommended_items) & set(actual_items))
        # 真实物品数
        total_actual += len(actual_items)
        # 推荐物品数
        total_recommend += len(recommended_items)

    recall = hit / total_actual if total_actual else 0
    precision = hit / total_recommend if total_recommend else 0
    return [recall, precision]