diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..35410ca
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/.idea/MarsCodeWorkspaceAppSettings.xml b/.idea/MarsCodeWorkspaceAppSettings.xml
new file mode 100644
index 0000000..8e12c96
--- /dev/null
+++ b/.idea/MarsCodeWorkspaceAppSettings.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000..312e865
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,21 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..98a0d17
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..adf2539
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/statistics_model2025.iml b/.idea/statistics_model2025.iml
new file mode 100644
index 0000000..cd2e5af
--- /dev/null
+++ b/.idea/statistics_model2025.iml
@@ -0,0 +1,12 @@
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..35eb1dd
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..63a8d62
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,32 @@
+# 数据处理
+
+3月26日 20:07 | 355字
+
+- 合并数据文件
+ - 合并热门数据
+ - 数据文件
+ - 视频: hot_data/分区/info
+ - up: hot_data/分区/up_info
+ - 弹幕/评论: hot_data/分区/BV号/...
+ - 单分区处理
+ - 按bv号匹配视频对应up指标，添加到info.csv
+ - 依序读取弹幕输出情感评分，(如果顺序没变的话) 直接添加一列“affective_scores”到info.csv
+ - 合并: 遍历分区info文件创建总文件，并给“视频荣誉”改名成“是否热门”并赋值为1
+ - 合并非热数据
+ - 同上，并赋值为0
+ - 合并两个文件
+ - 根据URL获取封面
+ - 按发布时间排序
+ - 文本数据合并
+ - 评论文本 (仅热门): 直接合并成列，人工筛选高质量文本
+ - 标签合并 (放一起整一个txt即可，拉个词云了事)
+ - 简介合并 (同上)
+- 数据预处理
+ - 是否为系列 (标题关键词分析)
+- 数据量化
+ - 大小分区名给一下，映射到大分区
+ - 视频分辨率: 360、720、1080、2k、4k、8k
+ - 封面处理并量化
+- 删除不用指标
+ - up主uid、bv号
+ - 视频简介、标签
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..0e5871e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,19 @@
+# 基础数据处理和分析
+pandas>=1.3.0
+numpy>=1.21.0
+
+# 情感分析相关
+torch>=1.10.0
+transformers>=4.26.0
+modelscope>=1.4.0
+snownlp>=0.12.3 # 新增的SnowNLP库
+
+# 词云和可视化
+matplotlib>=3.5.0
+wordcloud>=1.8.0
+jieba>=0.42.1
+Pillow>=9.0.0
+
+# 其他工具
+requests>=2.26.0
+tqdm>=4.45.0
diff --git a/弹幕情感评分.py b/弹幕情感评分.py
index 716effd..593192b 100644
--- a/弹幕情感评分.py
+++ b/弹幕情感评分.py
@@ -1,6 +1,6 @@
import pandas as pd
import torch
-from modelscope import AutoModelForSequenceClassification, AutoTokenizer
+from transformers import AutoModelForSequenceClassification, AutoTokenizer # 修改为从transformers导入
def load_data(file_path):
@@ -15,19 +15,24 @@ def load_data(file_path):
def analyze_sentiment(texts):
"""改进的情感分析函数"""
- # 使用模型配置
- model_name = "damo/nlp_structbert_sentiment-classification_chinese-base"
- tokenizer = AutoTokenizer.from_pretrained(model_name)
- model = AutoModelForSequenceClassification.from_pretrained(model_name)
+ # 使用新的模型配置
+ model_name = "IDEA-CCNL/Erlangshen-Roberta-330M-Sentiment"
+ try:
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ model = AutoModelForSequenceClassification.from_pretrained(model_name)
- # 批量处理提升效率
- inputs = tokenizer(texts, padding=True, truncation=True, max_length=128, return_tensors="pt")
- with torch.no_grad():
- outputs = model(**inputs)
+ # 批量处理提升效率
+ inputs = tokenizer(texts, padding=True, truncation=True, max_length=128, return_tensors="pt")
+ with torch.no_grad():
+ outputs = model(**inputs)
- # 调整概率计算方式
- probs = torch.softmax(outputs.logits, dim=1)
- return probs[:, 1].mean().item()
+ # 调整概率计算方式
+ probs = torch.softmax(outputs.logits, dim=1)
+ return probs[:, 1].mean().item() # 假设正例在位置1
+
+ except Exception as e:
+ print(f"模型加载失败: {str(e)}")
+ return 0.5 # 返回中性评分作为默认值
if __name__ == "__main__":